def test_peer_probe_invalid_ip_nonexist_host_nonexist_ip(self):
'''
Test script to verify peer probe non existing ip,
non_exsting_host and invalid-ip, peer probe has to
be fail for invalid-ip, non-existing-ip and
non existing host, verify Glusterd services up and
running or not after invalid peer probe,
and core file should not get created
under "/", /var/log/core and /tmp directory
'''
ret, test_timestamp, _ = g.run_local('date +%s')
test_timestamp = test_timestamp.strip()
g.log.info("Running Test : %s", self.id())
# Assigning non existing ip to variable
self.non_exist_ip = '256.256.256.256'
# Assigning invalid ip to variable
self.invalid_ip = '10.11.a'
# Assigning non existing host to variable
self.non_exist_host = 'abc.lab.eng.blr.redhat.com'
# Peer probe checks for non existing host
g.log.info("peer probe checking for non existing host")
ret, _, _ = peer_probe(self.mnode, self.non_exist_host)
self.assertNotEqual(
ret, 0, "peer probe should fail for "
"non existhost: %s" % self.non_exist_host)
g.log.info("peer probe failed for non existing host")
# Peer probe checks for invalid ip
g.log.info("peer probe checking for invalid ip")
ret, _, _ = peer_probe(self.mnode, self.invalid_ip)
self.assertNotEqual(
ret, 0, "peer probe shouldfail for "
"invalid ip: %s" % self.invalid_ip)
g.log.info("peer probe failed for invalid_ip")
# peer probe checks for non existing ip
g.log.info("peer probe checking for non existing ip")
ret, _, _ = peer_probe(self.mnode, self.non_exist_ip)
self.assertNotEqual(
ret, 0, "peer probe should fail for non exist "
"ip :%s" % self.non_exist_ip)
g.log.info("peer probe failed for non existing ip")
# Checks Glusterd services running or not after peer probe
# to invalid host and non existing host
self.mnode_list = []
self.mnode_list.append(self.mnode)
ret = is_glusterd_running(self.mnode_list)
self.assertEqual(ret, 0, "Glusterd service should be running")
# Chekcing core file created or not in "/", "/tmp" and
# "/var/log/core" directory
ret = is_core_file_created(self.servers, test_timestamp)
self.assertTrue(ret, "core file found")
def test_concurrent_set(self):
# time stamp of current test case
ret, test_timestamp, _ = g.run_local('date +%s')
test_timestamp = test_timestamp.strip()
# Create a volume
self.volname = "first-vol"
self.brick_list = form_bricks_list(self.mnode, self.volname, 3,
self.servers, self.all_servers_info)
ret = volume_create(self.mnode,
self.volname,
self.brick_list,
force=False)
self.assertEqual(ret[0], 0, ("Unable"
"to create volume % s" % self.volname))
g.log.info("Volume created successfuly % s" % self.volname)
# Create a volume
self.volname = "second-vol"
self.brick_list = form_bricks_list(self.mnode, self.volname, 3,
self.servers, self.all_servers_info)
g.log.info("Creating a volume")
ret = volume_create(self.mnode,
self.volname,
self.brick_list,
force=False)
self.assertEqual(ret[0], 0, ("Unable"
"to create volume % s" % self.volname))
g.log.info("Volume created successfuly % s" % self.volname)
cmd1 = ("for i in `seq 1 100`; do gluster volume set first-vol "
"read-ahead on; done")
cmd2 = ("for i in `seq 1 100`; do gluster volume set second-vol "
"write-behind on; done")
proc1 = g.run_async(random.choice(self.servers), cmd1)
proc2 = g.run_async(random.choice(self.servers), cmd2)
ret1, out1, err1 = proc1.async_communicate()
ret2, out2, err2 = proc2.async_communicate()
self.assertEqual(
ret1, 0, "Concurrent volume set on different volumes "
"simultaneously failed")
self.assertEqual(
ret2, 0, "Concurrent volume set on different volumes "
"simultaneously failed")
g.log.info("Setting options on different volumes @ same time "
"successfully completed")
ret = is_core_file_created(self.servers, test_timestamp)
if (ret):
g.log.info("No core file found, glusterd service "
"running successfully")
else:
g.log.error("core file found in directory, it "
"indicates the glusterd service crash")
self.assertTrue(ret, ("glusterd service should not crash"))
def test_invalid_memory_read_after_freed(self):
"""
Test case:
1. Create a volume and start it.
2. Mount the volume using FUSE.
3. Create multiple level of dirs and files inside every dir.
4. Rename files such that linkto files are created.
5. From the mount point do an rm -rf * and check if all files
are delete or not from mount point as well as backend bricks.
"""
# Fetch timestamp to check for core files
ret, test_timestamp, _ = g.run(self.mnode, "date +%s")
self.assertEqual(ret, 0, "date command failed")
test_timestamp = test_timestamp.strip()
# Create multiple level of dirs and files inside every dir
cmd = ("cd %s; for i in {1..100}; do mkdir dir$i; cd dir$i; "
"for i in {1..200}; do dd if=/dev/urandom of=file$i bs=1K"
" count=1; done; done" % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.first_client, cmd)
self.assertFalse(ret, "Failed to create dirs and files")
# Rename files such that linkto files are created
cmd = ("cd %s; for i in {1..100}; do cd dir$i; for i in {1..200}; do "
"mv file$i ntfile$i; done; done" % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.first_client, cmd)
self.assertFalse(ret, "Failed to rename files")
g.log.info("Files created and renamed successfully")
# From the mount point do an rm -rf * and check if all files
# are delete or not from mount point as well as backend bricks.
ret, _, _ = g.run(self.first_client,
"rm -rf {}/*".format(self.mounts[0].mountpoint))
self.assertFalse(ret, "rn -rf * failed on mount point")
ret = get_dir_contents(self.first_client,
"{}/".format(self.mounts[0].mountpoint))
self.assertEqual(
ret, [], "Unexpected: Files and directories still "
"seen from mount point")
for brick in get_all_bricks(self.mnode, self.volname):
node, brick_path = brick.split(":")
ret = get_dir_contents(node, "{}/".format(brick_path))
self.assertEqual(
ret, [], "Unexpected: Files and dirs still seen "
"on brick %s on node %s" % (brick_path, node))
g.log.info("rm -rf * on mount point successful")
# Check for core file on servers and clients
servers = self.servers + [self.first_client]
ret = is_core_file_created(servers, test_timestamp)
self.assertTrue(ret, "Core files found on servers used for test")
g.log.info("No cores found on all participating servers")
def test_enable_brickmux_create_and_stop_three_volumes(self):
"""
Test Case:
1.Set cluster.brick-multiplex to enabled.
2.Create three 1x3 replica volumes.
3.Start all the three volumes.
4.Stop three volumes one by one.
"""
# Timestamp of current test case of start time
ret, test_timestamp, _ = g.run_local('date +%s')
test_timestamp = test_timestamp.strip()
# Setting cluster.brick-multiplex to enable
ret = set_volume_options(self.mnode, 'all',
{'cluster.brick-multiplex': 'enable'})
self.assertTrue(ret, "Failed to set brick-multiplex to enable.")
g.log.info("Successfully set brick-multiplex to enable.")
# Create and start 3 volume
for number in range(1, 4):
self.volume['name'] = ("test_volume_%s" % number)
self.volname = ("test_volume_%s" % number)
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret,
"Failed to create and start %s" % self.volname)
g.log.info("Successfully created and started volume %s.",
self.volname)
# Checking brick process count.
for brick in get_all_bricks(self.mnode, self.volname):
server = brick.split(":")[0]
count = get_brick_processes_count(server)
self.assertEqual(
count, 1, "ERROR: More than one brick process on %s." % server)
g.log.info("Only one brick process present on %s", server)
# Stop three volumes one by one.
for number in range(1, 4):
self.volume['name'] = ("test_volume_%s" % number)
self.volname = ("test_volume_%s" % number)
ret, _, _ = volume_stop(self.mnode, self.volname)
self.assertEqual(ret, 0,
"Failed to stop the volume %s" % self.volname)
g.log.info("Volume %s stopped successfully", self.volname)
# Checking for core files.
ret = is_core_file_created(self.servers, test_timestamp)
self.assertTrue(ret, "Core file found.")
g.log.info("No core files found, glusterd service running "
"successfully")
def test_verify_peer_probe_with_firewall_ports_not_opened(self):
"""
Test Steps:
1. Open glusterd port only in Node1 using firewall-cmd command
2. Perform peer probe to Node2 from Node 1
3. Verify glusterd.log for Errors
4. Check for core files created
"""
ret, test_timestamp, _ = g.run_local('date +%s')
test_timestamp = test_timestamp.strip()
# Remove firewall service on the node to probe to
self._remove_firewall_service()
# Try peer probe from mnode to node
ret, _, err = peer_probe(self.mnode, self.node_to_probe)
self.assertEqual(ret, 1, ("Unexpected behavior: Peer probe should"
" fail when the firewall services are "
"down but returned success"))
expected_err = ('peer probe: failed: Probe returned with '
'Transport endpoint is not connected\n')
self.assertEqual(
err, expected_err,
"Expected error {}, but returned {}".format(expected_err, err))
msg = ("Peer probe of {} from {} failed as expected ".format(
self.mnode, self.node_to_probe))
g.log.info(msg)
# Verify there are no glusterd crashes
status = True
glusterd_logs = (self._get_test_specific_glusterd_log(
self.mnode).split("\n"))
for line in glusterd_logs:
if ' E ' in line:
status = False
g.log.info("Error found: ' %s '", line)
self.assertTrue(status, "Glusterd crash found")
# Verify no core files are created
ret = is_core_file_created(self.servers, test_timestamp)
self.assertTrue(ret, "Unexpected crash found.")
g.log.info("No core file found as expected")
def test_peer_probe_when_glusterd_down(self):
# pylint: disable=too-many-statements
'''
Test script to verify the behavior when we try to peer
probe a valid node whose glusterd is down
Also post validate to make sure no core files are created
under "/", /var/log/core and /tmp directory
Ref: BZ#1257394 Provide meaningful error on peer probe and peer detach
Test Steps:
1 check the current peer status
2 detach one of the valid nodes which is already part of cluster
3 stop glusterd on that node
4 try to attach above node to cluster, which must fail with
Transport End point error
5 Recheck the test using hostname, expected to see same result
6 start glusterd on that node
7 halt/reboot the node
8 try to peer probe the halted node, which must fail again.
9 The only error accepted is
"peer probe: failed: Probe returned with Transport endpoint is not
connected"
10 Check peer status and make sure no other nodes in peer reject state
'''
ret, test_timestamp, _ = g.run_local('date +%s')
test_timestamp = test_timestamp.strip()
# Detach one of the nodes which is part of the cluster
g.log.info("detaching server %s ", self.servers[1])
ret, _, err = peer_detach(self.mnode, self.servers[1])
msg = 'peer detach: failed: %s is not part of cluster\n' \
% self.servers[1]
if ret:
self.assertEqual(err, msg, "Failed to detach %s "
% (self.servers[1]))
# Bring down glusterd of the server which has been detached
g.log.info("Stopping glusterd on %s ", self.servers[1])
ret = stop_glusterd(self.servers[1])
self.assertTrue(ret, "Fail to stop glusterd on %s " % self.servers[1])
# Trying to peer probe the node whose glusterd was stopped using IP
g.log.info("Peer probing %s when glusterd down ", self.servers[1])
ret, _, err = peer_probe(self.mnode, self.servers[1])
self.assertNotEqual(ret, 0, "Peer probe should not pass when "
"glusterd is down")
self.assertEqual(err, "peer probe: failed: Probe returned with "
"Transport endpoint is not connected\n")
# Trying to peer probe the same node with hostname
g.log.info("Peer probing node %s using hostname with glusterd down ",
self.servers[1])
hostname = g.run(self.servers[1], "hostname")
ret, _, err = peer_probe(self.mnode, hostname[1].strip())
self.assertNotEqual(ret, 0, "Peer probe should not pass when "
"glusterd is down")
self.assertEqual(err, "peer probe: failed: Probe returned with"
" Transport endpoint is not connected\n")
# Start glusterd again for the next set of test steps
g.log.info("starting glusterd on %s ", self.servers[1])
ret = start_glusterd(self.servers[1])
self.assertTrue(ret, "glusterd couldn't start successfully on %s"
% self.servers[1])
# Bring down the network for sometime
network_status = bring_down_network_interface(self.servers[1], 150)
# Peer probing the node using IP when it is still not online
g.log.info("Peer probing node %s when network is down",
self.servers[1])
ret, _, err = peer_probe(self.mnode, self.servers[1])
self.assertNotEqual(ret, 0, "Peer probe passed when it was expected to"
" fail")
self.assertEqual(err.split("\n")[0], "peer probe: failed: Probe "
"returned with Transport endpoint"
" is not connected")
# Peer probing the node using hostname when it is still not online
g.log.info("Peer probing node %s using hostname which is still "
"not online ",
self.servers[1])
ret, _, err = peer_probe(self.mnode, hostname[1].strip())
self.assertNotEqual(ret, 0, "Peer probe should not pass when node "
"has not come online")
self.assertEqual(err.split("\n")[0], "peer probe: failed: Probe "
"returned with Transport endpoint"
" is not connected")
ret, _, _ = network_status.async_communicate()
if ret != 0:
g.log.error("Failed to perform network interface ops")
# Peer probe the node must pass
g.log.info("peer probing node %s", self.servers[1])
ret, _, err = peer_probe(self.mnode, self.servers[1])
self.assertEqual(ret, 0, "Peer probe has failed unexpectedly with "
"%s " % err)
# Checking if core file created in "/", "/tmp" and "/var/log/core"
ret = is_core_file_created(self.servers, test_timestamp)
self.assertTrue(ret, "core file found")
def test_volume_get(self):
"""
desc: performing different combinations of gluter
volume get functionalities
1. Create a gluster cluster
2. Get the option from the non-existing volume,
gluster volume get <non-existing vol> io-cache
3. Get all options from the non-existing volume,
gluster volume get <non-existing volume > all
4. Provide a incorrect command syntax to get the options
from the volume
gluster volume get <vol-name>
gluster volume get
gluster volume get io-cache
5. Create any type of volume in the cluster
6. Get the value of the non-existing option
gluster volume get <vol-name> temp.key
7. get all options set on the volume
gluster volume get <vol-name> all
8. get the specific option set on the volume
gluster volume get <vol-name> io-cache
9. Set an option on the volume
gluster volume set <vol-name> performance.low-prio-threads 14
10. Get all the options set on the volume and check
for low-prio-threads
gluster volume get <vol-name> all then get the
low-prio-threads value
11. Get all the options set on the volume
gluster volume get <vol-name> all
12. Check for any cores in "cd /"
"""
# pylint: disable=too-many-statements
# time stamp of current test case
ret, test_timestamp, _ = g.run_local('date +%s')
test_timestamp = test_timestamp.strip()
# performing gluster volume get command for non exist volume io-cache
self.non_exist_volume = "abc99"
ret, _, err = g.run(
self.mnode,
"gluster volume get %s io-cache" % self.non_exist_volume)
self.assertNotEqual(
ret, 0, "gluster volume get command should fail "
"for non existing volume with io-cache "
"option :%s" % self.non_exist_volume)
msg = ('Volume ' + self.non_exist_volume + ' does not exist')
self.assertIn(
msg, err, "No proper error message for non existing "
"volume with io-cache option :%s" % self.non_exist_volume)
g.log.info(
"gluster volume get command failed successfully for non "
"existing volume with io-cache option"
":%s", self.non_exist_volume)
# performing gluster volume get all command for non exist volume
ret, _, err = g.run(
self.mnode, "gluster volume get %s all" % self.non_exist_volume)
self.assertNotEqual(
ret, 0, "gluster volume get command should fail "
"for non existing volume %s with all "
"option" % self.non_exist_volume)
self.assertIn(
msg, err, "No proper error message for non existing "
"volume with all option:%s" % self.non_exist_volume)
g.log.info(
"gluster volume get command failed successfully for non "
"existing volume with all option :%s", self.non_exist_volume)
# performing gluster volume get command for non exist volume
ret, _, err = g.run(self.mnode, "gluster volume get "
"%s" % self.non_exist_volume)
self.assertNotEqual(
ret, 0, "gluster volume get command should "
"fail for non existing volume :%s" % self.non_exist_volume)
msg = 'get <VOLNAME|all> <key|all>'
self.assertIn(
msg, err, "No proper error message for non existing "
"volume :%s" % self.non_exist_volume)
g.log.info(
"gluster volume get command failed successfully for non "
"existing volume :%s", self.non_exist_volume)
# performing gluster volume get command without any volume name given
ret, _, err = g.run(self.mnode, "gluster volume get")
self.assertNotEqual(ret, 0, "gluster volume get command should fail")
self.assertIn(
msg, err, "No proper error message for gluster "
"volume get command")
g.log.info("gluster volume get command failed successfully")
# performing gluster volume get io-cache command
# without any volume name given
ret, _, err = g.run(self.mnode, "gluster volume get io-cache")
self.assertNotEqual(
ret, 0, "gluster volume get io-cache command "
"should fail")
self.assertIn(
msg, err, "No proper error message for gluster volume "
"get io-cache command")
g.log.info("gluster volume get io-cache command failed successfully")
# gluster volume get volname with non existing option
ret, _, err = g.run(self.mnode, "gluster volume "
"get %s temp.key" % self.volname)
self.assertNotEqual(
ret, 0, "gluster volume get command should fail "
"for existing volume %s with non-existing "
"option" % self.volname)
msg = 'Did you mean auth.allow or ...reject?'
self.assertIn(
msg, err, "No proper error message for existing "
"volume %s with non-existing option" % self.volname)
g.log.info(
"gluster volume get command failed successfully for "
"existing volume %s with non existing option", self.volname)
# performing gluster volume get volname all
ret = get_volume_options(self.mnode, self.volname, "all")
self.assertIsNotNone(
ret, "gluster volume get %s all command "
"failed" % self.volname)
g.log.info(
"gluster volume get %s all command executed "
"successfully", self.volname)
# performing gluster volume get volname io-cache
ret = get_volume_options(self.mnode, self.volname, "io-cache")
self.assertIsNotNone(
ret, "gluster volume get %s io-cache command "
"failed" % self.volname)
self.assertIn("on", ret['performance.io-cache'], "io-cache value "
"is not correct")
g.log.info("io-cache value is correct")
# Performing gluster volume set volname performance.low-prio-threads
prio_thread = {'performance.low-prio-threads': '14'}
ret = set_volume_options(self.mnode, self.volname, prio_thread)
self.assertTrue(
ret, "gluster volume set %s performance.low-prio-"
"threads failed" % self.volname)
g.log.info(
"gluster volume set %s "
"performance.low-prio-threads executed successfully", self.volname)
# Performing gluster volume get all, checking low-prio threads value
ret = get_volume_options(self.mnode, self.volname, "all")
self.assertIsNotNone(
ret, "gluster volume get %s all "
"failed" % self.volname)
self.assertIn("14", ret['performance.low-prio-threads'],
"performance.low-prio-threads value is not correct")
g.log.info("performance.low-prio-threads value is correct")
# performing gluster volume get volname all
ret = get_volume_options(self.mnode, self.volname, "all")
self.assertIsNotNone(
ret, "gluster volume get %s all command "
"failed" % self.volname)
g.log.info(
"gluster volume get %s all command executed "
"successfully", self.volname)
# Checking core file created or not in "/" directory
ret = is_core_file_created(self.servers, test_timestamp)
self.assertTrue(ret, "glusterd service should not crash")
g.log.info("No core file found, glusterd service "
"running successfully")
def test_peer_detach_host(self):
# peer Detaching specified server from cluster
# peer Detaching detached server again
# peer Detaching invalid host
# peer Detaching Non exist host
# peer Checking Core file created or not
# Peer detach one node which contains the bricks of volume created
# Peer detach force a node which is hosting bricks of a volume
# Timestamp of current test case of start time
ret, test_timestamp, _ = g.run_local('date +%s')
test_timestamp = test_timestamp.strip()
# Assigning non existing host to variable
self.non_exist_host = '256.256.256.256'
# Assigning invalid ip to vaiable
self.invalid_ip = '10.11.a'
# Peer detach to specified server
g.log.info("Start detach specified server :%s" % self.servers[1])
ret, out, _ = peer_detach(self.mnode, self.servers[1])
self.assertEqual(ret, 0,
"Failed to detach server :%s" % self.servers[1])
# Detached server detaching again, Expected to fail detach
g.log.info("Start detached server detaching "
"again : %s" % self.servers[1])
ret, out, _ = peer_detach(self.mnode, self.servers[1])
self.assertNotEqual(
ret, 0, "Detach server should "
"fail :%s" % self.servers[1])
# Probing detached server
g.log.info("Start probing detached server : %s" % self.servers[1])
ret = peer_probe_servers(self.mnode, self.servers[1])
self.assertTrue(
ret, "Peer probe failed from %s to other "
"server : %s" % (self.mnode, self.servers[1]))
# Detach invalid host
g.log.info("Start detaching invalid host :%s " % self.invalid_ip)
ret, out, _ = peer_detach(self.mnode, self.invalid_ip)
self.assertNotEqual(
ret, 0, "Detach invalid host should "
"fail :%s" % self.invalid_ip)
# Detach non exist host
g.log.info("Start detaching non exist host : %s" % self.non_exist_host)
ret, out, _ = peer_detach(self.mnode, self.non_exist_host)
self.assertNotEqual(
ret, 0, "Detach non existing host "
"should fail :%s" % self.non_exist_host)
# Chekcing core. file created or not in "/", "/tmp", "/log/var/core
# directory
ret = is_core_file_created(self.servers, test_timestamp)
self.assertTrue(ret, "glusterd service should not crash")
g.log.info("No core file found, glusterd service running "
"successfully")
# Creating Volume
g.log.info("Started creating volume: %s" % self.volname)
ret = self.setup_volume()
self.assertTrue(ret, "Volume creation failed: %s" % self.volname)
# Peer detach one node which contains the bricks of the volume created
g.log.info("Start detaching server %s which is hosting "
"bricks of a volume" % self.servers[1])
ret, out, err = peer_detach(self.mnode, self.servers[1])
self.assertNotEqual(ret, 0,
"detach server should fail: %s" % self.servers[1])
msg = ('peer detach: failed: Brick(s) with the peer ' +
self.servers[1] + ' ' + 'exist in cluster')
self.assertIn(msg, err, "Peer detach not failed with "
"proper error message")
# Peer detach force a node which is hosting bricks of a volume
g.log.info("start detaching server %s with force option "
"which is hosting bricks of a volume" % self.servers[1])
ret, out, err = peer_detach(self.mnode, self.servers[1], force=True)
self.assertNotEqual(
ret, 0, "detach server should fail with force "
"option : %s" % self.servers[1])
msg = ('peer detach: failed: Brick(s) with the peer ' +
self.servers[1] + ' ' + 'exist in cluster')
self.assertIn(
msg, err, "Peer detach not failed with proper "
"error message with force option")
def test_peer_probe_when_glusterd_down(self):
# pylint: disable=too-many-statements
'''
Test script to verify the behavior when we try to peer
probe a valid node whose glusterd is down
Also post validate to make sure no core files are created
under "/", /var/log/core and /tmp directory
Ref: BZ#1257394 Provide meaningful error on peer probe and peer detach
Test Steps:
1 check the current peer status
2 detach one of the valid nodes which is already part of cluster
3 stop glusterd on that node
4 try to attach above node to cluster, which must fail with
Transport End point error
5 Recheck the test using hostname, expected to see same result
6 start glusterd on that node
7 halt/reboot the node
8 try to peer probe the halted node, which must fail again.
9 The only error accepted is
"peer probe: failed: Probe returned with Transport endpoint is not
connected"
10 Check peer status and make sure no other nodes in peer reject state
'''
ret, test_timestamp, _ = g.run_local('date +%s')
test_timestamp = test_timestamp.strip()
# detach one of the nodes which is part of the cluster
g.log.info("detaching server %s ", self.servers[1])
ret, _, err = peer_detach(self.mnode, self.servers[1])
msg = 'peer detach: failed: %s is not part of cluster\n' \
% self.servers[1]
if ret:
self.assertEqual(err, msg, "Failed to detach %s "
% (self.servers[1]))
# bring down glusterd of the server which has been detached
g.log.info("Stopping glusterd on %s ", self.servers[1])
ret = stop_glusterd(self.servers[1])
self.assertTrue(ret, "Fail to stop glusterd on %s " % self.servers[1])
# trying to peer probe the node whose glusterd was stopped using its IP
g.log.info("Peer probing %s when glusterd down ", self.servers[1])
ret, _, err = peer_probe(self.mnode, self.servers[1])
self.assertNotEqual(ret, 0, "Peer probe should not pass when "
"glusterd is down")
self.assertEqual(err, "peer probe: failed: Probe returned with "
"Transport endpoint is not connected\n")
# trying to peer probe the same node with hostname
g.log.info("Peer probing node %s using hostname with glusterd down ",
self.servers[1])
hostname = g.run(self.servers[1], "hostname")
ret, _, err = peer_probe(self.mnode, hostname[1].strip())
self.assertNotEqual(ret, 0, "Peer probe should not pass when "
"glusterd is down")
self.assertEqual(err, "peer probe: failed: Probe returned with"
" Transport endpoint is not connected\n")
# start glusterd again for the next set of test steps
g.log.info("starting glusterd on %s ", self.servers[1])
ret = start_glusterd(self.servers[1])
self.assertTrue(ret, "glusterd couldn't start successfully on %s"
% self.servers[1])
# reboot a server and then trying to peer probe at the time of reboot
g.log.info("Rebooting %s and checking peer probe", self.servers[1])
reboot = g.run_async(self.servers[1], "reboot")
# Mandatory sleep for 3 seconds to make sure node is in halted state
sleep(3)
# Peer probing the node using IP when it is still not online
g.log.info("Peer probing node %s which has been issued a reboot ",
self.servers[1])
ret, _, err = peer_probe(self.mnode, self.servers[1])
self.assertNotEqual(ret, 0, "Peer probe passed when it was expected to"
" fail")
self.assertEqual(err, "peer probe: failed: Probe returned with "
"Transport endpoint is not connected\n")
# Peer probing the node using hostname when it is still not online
g.log.info("Peer probing node %s using hostname which is still "
"not online ",
self.servers[1])
ret, _, err = peer_probe(self.mnode, hostname[1].strip())
self.assertNotEqual(ret, 0, "Peer probe should not pass when node "
"has not come online")
self.assertEqual(err, "peer probe: failed: Probe returned with "
"Transport endpoint is not connected\n")
ret, _, _ = reboot.async_communicate()
self.assertEqual(ret, 255, "reboot failed")
# Validate if rebooted node is online or not
count = 0
while count < 40:
sleep(15)
ret, _ = are_nodes_online(self.servers[1])
if ret:
g.log.info("Node %s is online", self.servers[1])
break
count += 1
self.assertTrue(ret, "Node in test not yet online")
# check if glusterd is running post reboot
ret = wait_for_glusterd_to_start(self.servers[1],
glusterd_start_wait_timeout=120)
self.assertTrue(ret, "Glusterd service is not running post reboot")
# peer probe the node must pass
g.log.info("peer probing node %s", self.servers[1])
ret, _, err = peer_probe(self.mnode, self.servers[1])
self.assertEqual(ret, 0, "Peer probe has failed unexpectedly with "
"%s " % err)
# checking if core file created in "/", "/tmp" and "/var/log/core"
ret = is_core_file_created(self.servers, test_timestamp)
self.assertTrue(ret, "core file found")
def test_profile_operations(self):
"""
Test Case:
1) Create a volume and start it.
2) Mount volume on client and start IO.
3) Start profile on the volume.
4) Run profile info and see if all bricks are present or not.
6) Create another volume.
7) Run profile info without starting the profile.
8) Run profile info with all possible options without starting
the profile.
"""
# Timestamp of current test case of start time
ret, test_timestamp, _ = g.run_local('date +%s')
test_timestamp = test_timestamp.strip()
# Start IO on mount points.
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
counter = 1
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dir-depth 4 "
"--dir-length 6 "
"--dirname-start-num %d "
"--max-num-of-dirs 3 "
"--num-of-files 5 %s" %
(self.script_upload_path, counter, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
counter += 1
# Start profile on volume.
ret, _, _ = profile_start(self.mnode, self.volname)
self.assertEqual(
ret, 0, "Failed to start profile on volume: %s" % self.volname)
g.log.info("Successfully started profile on volume: %s", self.volname)
# Getting and checking output of profile info.
ret, out, _ = profile_info(self.mnode, self.volname)
self.assertEqual(
ret, 0, "Failed to run profile info on volume: %s" % self.volname)
g.log.info("Successfully executed profile info on volume: %s",
self.volname)
# Checking if all bricks are present in profile info.
brick_list = get_all_bricks(self.mnode, self.volname)
for brick in brick_list:
self.assertTrue(
brick in out,
"Brick %s not a part of profile info output." % brick)
g.log.info("Brick %s showing in profile info output.", brick)
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
g.log.info("IO validation complete.")
# Create and start a volume
self.volume['name'] = "volume_2"
self.volname = "volume_2"
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start volume")
g.log.info("Successfully created and started volume_2")
# Check profile info on volume without starting profile
ret, _, _ = profile_info(self.mnode, self.volname)
self.assertNotEqual(
ret, 0, "Unexpected:Successfully ran profile info"
" on volume: %s" % self.volname)
g.log.info("Expected: Failed to run pofile info on volume: %s",
self.volname)
# Running profile info with different profile options.
profile_options = ('peek', 'incremental', 'clear', 'incremental peek',
'cumulative')
for option in profile_options:
# Getting and checking output of profile info.
ret, _, _ = profile_info(self.mnode, self.volname, options=option)
self.assertNotEqual(
ret, 0, "Unexpected: Successfully ran profile info"
" %s on volume: %s" % (option, self.volname))
g.log.info(
"Expected: Failed to execute profile info %s on"
" volume: %s", option, self.volname)
# Chekcing for core files.
ret = is_core_file_created(self.servers, test_timestamp)
self.assertTrue(ret, "glusterd service should not crash")
g.log.info("No core file found, glusterd service running "
"successfully")
# Checking whether glusterd is running or not
ret = is_glusterd_running(self.servers)
self.assertEqual(ret, 0, "Glusterd has crashed on nodes.")
g.log.info("No glusterd crashes observed.")
def test_ec_truncate_file_with_brick_down(self):
"""
Test steps:
1. Create a volume, start and mount it on a client
2. Bring down redundant bricks in the subvol
3. Create a file on the volume using "touch"
4. Truncate the file using "O_TRUNC"
5. Bring the brick online
6. Write data on the file and wait for heal completion
7. Check for crashes and coredumps
"""
# pylint: disable=unsubscriptable-object
for restart_type in ("volume_start", "node_reboot"):
# Time stamp from mnode for checking cores at the end of test
ret, test_timestamp, _ = g.run(self.mnode, "date +%s")
self.assertEqual(ret, 0, "date command failed")
test_timestamp = test_timestamp.strip()
# Create a file using touch
file_name = self.mounts[0].mountpoint + "/test_1"
ret, _, err = g.run(self.mounts[0].client_system,
"touch {}".format(file_name))
self.assertEqual(ret, 0, "File creation failed")
g.log.info("File Created successfully")
# List two bricks in each subvol
subvols = get_subvols(self.mnode, self.volname)['volume_subvols']
bricks_to_bring_offline = []
for subvol in subvols:
self.assertTrue(subvol, "List is empty")
bricks_to_bring_offline.extend(sample(subvol, 2))
# Bring two bricks of each subvol offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, "Bricks are still online")
# Validating the bricks are offline or not
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, "Few of the bricks are still online in"
" {} in".format(bricks_to_bring_offline))
# Truncate the file
cmd = (
'python -c "import os, sys; fd = os.open(\'{}\', os.O_TRUNC )'
'; os.close( fd )"').format(file_name)
ret, _, err = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, err)
g.log.info("File truncated successfully")
# Bring back the bricks online
if restart_type == "volume_start":
# Bring back bricks online by volume start
ret, _, err = volume_start(self.mnode,
self.volname,
force=True)
self.assertEqual(ret, 0, err)
g.log.info("All bricks are online")
elif restart_type == "node_reboot":
# Bring back the bricks online by node restart
for brick in bricks_to_bring_offline:
node_to_reboot = brick.split(":")[0]
ret = reboot_nodes_and_wait_to_come_online(node_to_reboot)
self.assertTrue(
ret, "Reboot Failed on node: "
"{}".format(node_to_reboot))
g.log.info("Node: %s rebooted successfully",
node_to_reboot)
time.sleep(60)
# Check whether bricks are online or not
ret = are_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret,
"Bricks {} are still offline".format(bricks_to_bring_offline))
# write data to the file
cmd = ('python -c "import os, sys;fd = os.open(\'{}\', '
'os.O_RDWR) ;'
'os.write(fd, \'This is test after truncate\'.encode());'
' os.close(fd)"').format(file_name)
ret, _, err = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, err)
g.log.info("Data written successfully on to the file")
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, "Heal pending for file {}".format(file_name))
# check for any crashes on servers and client
for nodes in (self.servers, [self.clients[0]]):
ret = is_core_file_created(nodes, test_timestamp)
self.assertTrue(ret,
"Cores found on the {} nodes".format(nodes))
def test_peer_detach_host(self):
# pylint: disable = too-many-statements
# peer Detaching specified server from cluster
# peer Detaching detached server again and checking the error msg
# peer Detaching invalid host
# peer Detaching Non exist host
# peer Checking Core file created or not
# Peer detach one node which contains the bricks of volume created
# Peer detach force a node which is hosting bricks of a volume
# Peer detach one node which hosts bricks of offline volume
# Peer detach force a node which hosts bricks of offline volume
# Timestamp of current test case of start time
ret, test_timestamp, _ = g.run_local('date +%s')
test_timestamp = test_timestamp.strip()
# Assigning non existing host to variable
self.non_exist_host = '256.256.256.256'
# Assigning invalid ip to variable
self.invalid_ip = '10.11.a'
# Peer detach to specified server
g.log.info("Start detach specified server :%s", self.servers[1])
ret, _, _ = peer_detach(self.mnode, self.servers[1])
self.assertEqual(ret, 0,
"Failed to detach server :%s" % self.servers[1])
# Detached server detaching again, Expected to fail detach
g.log.info("Start detached server detaching "
"again : %s", self.servers[1])
ret, _, err = peer_detach(self.mnode, self.servers[1])
self.assertNotEqual(
ret, 0, "Detach server should "
"fail :%s" % self.servers[1])
self.assertEqual(
err, "peer detach: failed: %s is not part of "
"cluster\n" % self.servers[1], "Peer "
"Detach didn't fail as expected")
# Probing detached server
g.log.info("Start probing detached server : %s", self.servers[1])
ret = peer_probe_servers(self.mnode, self.servers[1])
self.assertTrue(
ret, "Peer probe failed from %s to other "
"server : %s" % (self.mnode, self.servers[1]))
# Detach invalid host
g.log.info("Start detaching invalid host :%s ", self.invalid_ip)
ret, _, _ = peer_detach(self.mnode, self.invalid_ip)
self.assertNotEqual(
ret, 0, "Detach invalid host should "
"fail :%s" % self.invalid_ip)
# Detach non exist host
g.log.info("Start detaching non exist host : %s", self.non_exist_host)
ret, _, _ = peer_detach(self.mnode, self.non_exist_host)
self.assertNotEqual(
ret, 0, "Detach non existing host "
"should fail :%s" % self.non_exist_host)
# Creating Volume
g.log.info("Started creating volume: %s", self.volname)
ret = self.setup_volume()
self.assertTrue(ret, "Volume creation failed: %s" % self.volname)
# Peer detach one node which contains the bricks of the volume created
g.log.info(
"Start detaching server %s which is hosting "
"bricks of a volume", self.servers[1])
self.check_detach_error_message(use_force=False)
# Peer detach force a node which is hosting bricks of a volume
g.log.info(
"Start detaching server using force %s which is hosting "
"bricks of a volume", self.servers[1])
self.check_detach_error_message()
# Peer detach one node which contains bricks of an offline volume
g.log.info("stopping the volume")
ret, _, err = volume_stop(self.mnode, self.volname)
msg = ('volume stop: ' + 'self.volname' + ': failed: Volume ' +
'self.volname' + ' is not in the started state\n')
if msg not in err:
self.assertEqual(ret, 0,
"stopping volume %s failed" % self.volname)
g.log.info(
"Start to detach server %s which is hosting "
"bricks of an offline volume", self.servers[1])
self.check_detach_error_message(use_force=False)
# Forceful Peer detach node which hosts bricks of offline volume
g.log.info(
"start detaching server %s with force option "
"which is hosting bricks of a volume", self.servers[1])
self.check_detach_error_message()
# starting volume for proper cleanup
ret, _, _ = volume_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "volume start failed")
# Checking core. file created or not in "/", "/tmp", "/log/var/core
# directory
ret = is_core_file_created(self.servers, test_timestamp)
self.assertTrue(ret, "glusterd service should not crash")
g.log.info("No core file found, glusterd service running "
"successfully")
def test_profile_simultaneously_on_different_nodes(self):
"""
Test Case:
1) Create a volume and start it.
2) Mount volume on client and start IO.
3) Start profile on the volume.
4) Create another volume.
5) Start profile on the volume.
6) Run volume status in a loop in one of the node.
7) Run profile info for the new volume on one of the other node
8) Run profile info for the new volume in loop for 100 times on
the other node
"""
# Timestamp of current test case of start time
ret, test_timestamp, _ = g.run_local('date +%s')
test_timestamp = test_timestamp.strip()
# Start IO on mount points.
self.all_mounts_procs = []
counter = 1
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dir-depth 4 "
"--dir-length 6 "
"--dirname-start-num %d "
"--max-num-of-dirs 3 "
"--num-of-files 5 %s" %
(self.script_upload_path, counter, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
counter += 1
# Start profile on volume.
ret, _, _ = profile_start(self.mnode, self.volname)
self.assertEqual(
ret, 0, "Failed to start profile on volume: %s" % self.volname)
g.log.info("Successfully started profile on volume: %s", self.volname)
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
g.log.info("IO validation complete.")
# Create and start a volume
self.volume['name'] = "volume_2"
self.volname = "volume_2"
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start volume")
g.log.info("Successfully created and started volume_2")
# Start profile on volume.
ret, _, _ = profile_start(self.mnode, self.volname)
self.assertEqual(
ret, 0, "Failed to start profile on volume: %s" % self.volname)
g.log.info("Successfully started profile on volume: %s", self.volname)
# Run volume status on one of the node in loop
cmd = "for i in `seq 1 100`;do gluster v status;done"
proc1 = g.run_async(self.servers[1], cmd)
# Check profile on one of the other node
cmd = "gluster v profile %s info" % self.volname
ret, _, _ = g.run(self.mnode, cmd)
self.assertEqual(
ret, 0, "Failed to run profile info on volume: %s"
" on node %s" % (self.volname, self.mnode))
g.log.info("Successfully run pofile info on volume: %s on node %s",
self.volname, self.mnode)
# Run volume profile info on one of the other node in loop
cmd = """for i in `seq 1 100`;do gluster v profile %s info;
done""" % self.volname
proc2 = g.run_async(self.servers[3], cmd)
ret1, _, _ = proc1.async_communicate()
ret2, _, _ = proc2.async_communicate()
self.assertEqual(
ret1, 0, "Failed to run volume status in a loop"
" on node %s" % self.servers[1])
g.log.info(
"Successfully running volume status in a loop on node"
" %s", self.servers[1])
self.assertEqual(
ret2, 0, "Failed to run profile info in a loop"
" on node %s" % self.servers[3])
g.log.info(
"Successfully running volume status in a loop on node"
" %s", self.servers[3])
# Chekcing for core files.
ret = is_core_file_created(self.servers, test_timestamp)
self.assertTrue(ret, "glusterd service should not crash")
g.log.info("No core file found, glusterd service running "
"successfully")
# Checking whether glusterd is running or not
ret = is_glusterd_running(self.servers)
self.assertEqual(ret, 0, "Glusterd has crashed on nodes.")
g.log.info("No glusterd crashes observed.")