def setUp(self): GlusterBaseClass.setUp.im_func(self) # check whether peers are in connected state ret = self.validate_peers_are_connected() if not ret: raise ExecutionError("Peers are not in connected state") # detach all the nodes ret = peer_detach_servers(self.mnode, self.servers) if not ret: raise ExecutionError("Peer detach failed to all the servers from " "the node.") g.log.info("Peer detach SUCCESSFUL.") # Uploading file_dir script in all client direcotries g.log.info( "Upload io scripts to clients %s for running IO on " "mounts", self.clients) script_local_path = ("/usr/share/glustolibs/io/scripts/" "file_dir_ops.py") self.script_upload_path = ("/usr/share/glustolibs/io/scripts/" "file_dir_ops.py") ret = upload_scripts(self.clients, script_local_path) if not ret: raise ExecutionError("Failed to upload IO scripts to clients %s" % self.clients) g.log.info("Successfully uploaded IO scripts to clients %s", self.clients)
def setUp(self): # Performing peer detach ret = peer_detach_servers(self.mnode, self.servers) if not ret: raise ExecutionError("Failed to detach servers %s" % self.servers) g.log.info("Peer detach SUCCESSFUL.") self.get_super_method(self, 'setUp')()
def setUp(self): # Performing peer detach ret = peer_detach_servers(self.mnode, self.servers) if not ret: raise ExecutionError("Failed to detach servers %s" % self.servers) g.log.info("Peer detach SUCCESSFUL.") GlusterBaseClass.setUp.im_func(self)
def setUp(self): self.get_super_method(self, 'setUp')() # Performing peer detach if not peer_detach_servers(self.mnode, self.servers): raise ExecutionError("Failed to detach servers %s" % self.servers) g.log.info("Peer detach SUCCESSFUL.") self.peers_in_pool = [] self.by_type = "" self.node = None
def setUpClass(cls): """ """ # Read all the cluster config from the g.config and assign it to # class variables GlusterBaseClass.setUpClass.im_func(cls) # Detach all the servers if it's already attached to the cluster nodes_in_pool_list = nodes_from_pool_list(cls.mnode) if nodes_in_pool_list is None: g.log.error( "Unable to get nodes from gluster pool list " "from node %s", cls.mnode) else: g.log.info("Nodes in pool: %s", nodes_in_pool_list) if nodes_in_pool_list: if cls.mnode in nodes_in_pool_list: nodes_in_pool_list.remove(cls.mnode) g.log.info("Detaching servers '%s' from the cluster from node %s", nodes_in_pool_list, cls.mnode) ret = peer_detach_servers(cls.mnode, nodes_in_pool_list) if not ret: raise ExecutionError( "Failed to detach some or all " "servers %s from the cluster " "from node %s", nodes_in_pool_list, cls.mnode) g.log.info( "Successfully detached all servers '%s' " "from the cluster from node %s", nodes_in_pool_list, cls.mnode) # Get pool list from mnode g.log.info("Pool list on node %s", cls.mnode) ret, out, err = pool_list(cls.mnode) if ret != 0: raise ExecutionError("Failed to get pool list on node %s: %s", cls.mnode, err) g.log.info("Successfully got pool list on node %s:\n%s", cls.mnode, out) # Get peer status output from all servers for server in cls.servers: g.log.info("Peer status on node %s", server) ret, out, err = peer_status(server) if ret != 0: raise ExecutionError( "Failed to get peer status on node %s: " "%s", server, err) g.log.info("Successfully got peer status on node %s:\n%s", server, out)
def setUp(self): self.get_super_method(self, 'setUp')() # check whether peers are in connected state ret = self.validate_peers_are_connected() if not ret: raise ExecutionError("Peers are not in connected state") # detach all the nodes ret = peer_detach_servers(self.mnode, self.servers) if not ret: raise ExecutionError("Peer detach failed to all the servers from " "the node %s." % self.mnode) g.log.info("Peer detach SUCCESSFUL.")
def setUp(self): GlusterBaseClass.setUp.im_func(self) # check whether peers are in connected state ret = self.validate_peers_are_connected() if not ret: raise ExecutionError("Peers are not in connected state") # detach all the nodes ret = peer_detach_servers(self.mnode, self.servers) if not ret: raise ExecutionError("Peer detach failed to all the servers from " "the node %s." % self.mnode) g.log.info("Peer detach SUCCESSFUL.")
def setUp(self): self.get_super_method(self, 'setUp')() # check whether peers are in connected state ret = self.validate_peers_are_connected() if not ret: raise ExecutionError("Peers are not in connected state") g.log.info("Peers are in connected state") # detach a node from cluster, assume last node last_node = self.servers[len(self.servers) - 1] ret = peer_detach_servers(self.mnode, last_node) if not ret: raise ExecutionError("Peer detach failed to the last node " "%s from %s" % (last_node, self.mnode)) g.log.info("Peer detach SUCCESSFUL.")
def test_logs_while_peer_detach(self): ''' -> Detach the node from peer -> Check that any error messages related to peer detach in glusterd log file -> No errors should be there in glusterd log file ''' # Getting timestamp _, timestamp, _ = g.run_local('date +%s') timestamp = timestamp.strip() # glusterd logs ret, _, _ = g.run(self.mnode, 'cp /var/log/glusterfs/glusterd.log ' '/var/log/glusterfs/glusterd_%s.log' % timestamp) if ret: raise ExecutionError("Failed to copy glusterd logs") # Clearing the existing glusterd log file ret, _, _ = g.run(self.mnode, 'echo > /var/log/glusterfs/glusterd.log') if ret: raise ExecutionError("Failed to clear glusterd.log file on %s" % self.mnode) # Performing peer detach self.random_server = random.choice(self.servers[1:]) ret = peer_detach_servers(self.mnode, self.random_server) self.assertTrue(ret, "Failed to detach peer %s" % self.random_server) g.log.info("Peer detach successful for %s", self.random_server) # Searching for error message in log ret, out, _ = g.run( self.mnode, "grep ' E ' /var/log/glusterfs/glusterd.log | wc -l") self.assertEqual(ret, 0, "Failed to get error message count in " "glusterd log file") g.log.info("Successful getting error message count in log file") self.assertEqual(int(out), 0, "Found Error messages in glusterd log " "file after peer detach") g.log.info("No error messages found in gluterd log file after peer " "detach")
def setUpClass(cls): # Calling GlusterBaseClass setUpClass cls.get_super_method(cls, 'setUpClass')() ret = volume_exists(cls.mnode, cls.volname) if ret: ret = cleanup_volume(cls.mnode, cls.volname) if not ret: raise ExecutionError("Unable to delete volume") g.log.info("Successfully deleted volume % s", cls.volname) # Check if peer is connected state or not and detach all the nodes for server in cls.servers: ret = is_peer_connected(server, cls.servers) if ret: ret = peer_detach_servers(server, cls.servers) if not ret: raise ExecutionError( "Detach failed from all the servers from the node.") g.log.info("Peer detach SUCCESSFUL.") # Before starting the testcase, proceed only it has minimum of 4 nodes if len(cls.servers) < 4: raise ExecutionError("Minimun four nodes required for this " " testcase to execute")
def test_volume_create(self): # create and start a volume self.volume['name'] = "first_volume" self.volname = "first_volume" ret = setup_volume(self.mnode, self.all_servers_info, self.volume) self.assertTrue(ret, "Failed to create and start volume") # bring a brick down and volume start force should bring it to online g.log.info("Get all the bricks of the volume") bricks_list = get_all_bricks(self.mnode, self.volname) self.assertIsNotNone(bricks_list, "Failed to get the brick list") g.log.info("Successfully got the list of bricks of volume") ret = bring_bricks_offline(self.volname, bricks_list[0:2]) self.assertTrue(ret, "Failed to bring down the bricks") g.log.info("Successfully brought the bricks down") ret, _, _ = volume_start(self.mnode, self.volname, force=True) self.assertEqual(ret, 0, "Failed to start the volume") g.log.info("Volume start with force is success") ret = wait_for_bricks_to_be_online(self.mnode, self.volname) self.assertTrue(ret, "Failed to bring the bricks online") g.log.info("Volume start with force successfully brought all the " "bricks online") # create volume with previously used bricks and different volume name self.volname = "second_volume" ret, _, _ = volume_create(self.mnode, self.volname, bricks_list) self.assertNotEqual( ret, 0, "Expected: It should fail to create a " "volume with previously used bricks. Actual:" "Successfully created the volume with previously" " used bricks") g.log.info("Failed to create the volume with previously used bricks") # create a volume with already existing volume name self.volume['name'] = "first_volume" ret = setup_volume(self.mnode, self.all_servers_info, self.volume) self.assertTrue( ret, "Expected: It should fail to create a volume" " with already existing volume name. Actual: " "Successfully created the volume with " "already existing volname") g.log.info("Failed to create the volume with already existing volname") # creating a volume with non existing brick path should fail self.volname = "second_volume" bricks_list = form_bricks_list(self.mnode, self.volname, len(self.servers), self.servers, self.all_servers_info) nonexisting_brick_index = random.randint(0, len(bricks_list) - 1) non_existing_brick = bricks_list[nonexisting_brick_index].split(":")[0] non_existing_path = ":/brick/non_existing_path" non_existing_brick = non_existing_brick + non_existing_path bricks_list[nonexisting_brick_index] = non_existing_brick ret, _, _ = volume_create(self.mnode, self.volname, bricks_list) self.assertNotEqual( ret, 0, "Expected: Creating a volume with non " "existing brick path should fail. Actual: " "Successfully created the volume with " "non existing brick path") g.log.info("Failed to create the volume with non existing brick path") # cleanup the volume and peer detach all servers. form two clusters,try # to create a volume with bricks whose nodes are in different clusters # cleanup volumes vol_list = get_volume_list(self.mnode) self.assertIsNotNone(vol_list, "Failed to get the volume list") for volume in vol_list: ret = cleanup_volume(self.mnode, volume) self.assertTrue(ret, "Unable to delete volume % s" % volume) # peer detach all servers ret = peer_detach_servers(self.mnode, self.servers) self.assertTrue(ret, "Peer detach to all servers is failed") g.log.info("Peer detach to all the servers is success") # form cluster 1 ret, _, _ = peer_probe(self.servers[0], self.servers[1]) self.assertEqual( ret, 0, "Peer probe from %s to %s is failed" % (self.servers[0], self.servers[1])) g.log.info("Peer probe is success from %s to %s" % (self.servers[0], self.servers[1])) # form cluster 2 ret, _, _ = peer_probe(self.servers[2], self.servers[3]) self.assertEqual( ret, 0, "Peer probe from %s to %s is failed" % (self.servers[2], self.servers[3])) g.log.info("Peer probe is success from %s to %s" % (self.servers[2], self.servers[3])) # Creating a volume with bricks which are part of another # cluster should fail ret = setup_volume(self.mnode, self.all_servers_info, self.volume) self.assertFalse( ret, "Expected: Creating a volume with bricks" " which are part of another cluster should fail." " Actual: Successfully created the volume with " "bricks which are part of another cluster") g.log.info("Failed to create the volume with bricks which are " "part of another cluster") # form a cluster, bring a node down. try to create a volume when one of # the brick node is down ret, _, _ = peer_detach(self.servers[2], self.servers[3]) self.assertEqual(ret, 0, "Peer detach is failed") g.log.info("Peer detach is success") ret = peer_probe_servers(self.mnode, self.servers) self.assertTrue(ret, "Peer probe is failed") g.log.info("Peer probe to all the servers is success") random_server = self.servers[random.randint(1, len(self.servers) - 1)] ret = stop_glusterd(random_server) self.assertTrue(ret, "Glusterd is stopped successfully") self.volume['name'] = "third_volume" ret = setup_volume(self.mnode, self.all_servers_info, self.volume) self.assertFalse( ret, "Expected: It should fail to create a volume " "when one of the node is down. Actual: Successfully " "created the volume with bbrick whose node is down") g.log.info("Failed to create the volume with brick whose node is down")
def test_glustershd_on_newly_probed_server(self): """ Test script to verify glustershd process on newly probed server * check glustershd process - only 1 glustershd process should be running * Add new node to cluster * check glustershd process - only 1 glustershd process should be running on all servers inclusing newly probed server * stop the volume * add another node to cluster * check glustershd process - glustershd process shouldn't be running on servers including newly probed server * start the volume * check glustershd process - only 1 glustershd process should be running on all servers inclusing newly probed server """ # pylint: disable=too-many-statements nodes = self.volume['servers'][:-2] # check the self-heal daemon process g.log.info("Starting to get self heal daemon process on " "nodes %s", nodes) ret, pids = get_self_heal_daemon_pid(nodes) self.assertTrue(ret, ("Either no self heal daemon process found or " "more than one self heal daemon process " "found : %s" % pids)) g.log.info( "Successful in getting single self heal daemon process" " on all nodes %s", nodes) # Add new node to the cluster g.log.info("Peer probe for %s", self.extra_servers[0]) ret = peer_probe_servers(self.mnode, self.extra_servers[0]) self.assertTrue( ret, "Failed to peer probe server : %s" % self.extra_servers[0]) g.log.info( "Peer probe success for %s and all peers are in " "connected state", self.extra_servers[0]) nodes.append(self.extra_servers[0]) # check the self-heal daemon process and it should be running on # newly probed servers g.log.info("Starting to get self-heal daemon process on " "nodes %s", nodes) ret, pids = get_self_heal_daemon_pid(nodes) self.assertTrue(ret, ("Either no self heal daemon process found or " "more than one self heal daemon process " "found : %s" % pids)) g.log.info( "Successful in getting single self heal daemon process" " on all nodes %s", nodes) # stop the volume g.log.info("Stopping the volume %s", self.volname) ret = volume_stop(self.mnode, self.volname) self.assertTrue(ret, ("Failed to stop volume %s" % self.volname)) g.log.info("Successfully stopped volume %s", self.volname) # Add another new node to the cluster g.log.info("peer probe for %s", self.extra_servers[1]) ret = peer_probe_servers(self.mnode, self.extra_servers[1]) self.assertTrue( ret, "Failed to peer probe server : %s" % self.extra_servers[1]) g.log.info( "Peer probe success for %s and all peers are in " "connected state", self.extra_servers[1]) nodes.append(self.extra_servers[1]) # check the self-heal daemon process after stopping volume and # no self heal daemon should be running including newly probed node g.log.info("Starting to get self-heal daemon process on " "nodes %s", nodes) ret, pids = get_self_heal_daemon_pid(nodes) self.assertFalse(ret, ("Self Heal Daemon process is running even " "after stopping volume %s" % self.volname)) for node in pids: self.assertEquals(pids[node][0], -1, ("Self Heal Daemon is still " "running on node %s even " "after stopping all " "volumes" % node)) g.log.info("Expected : No self heal daemon process is running " "after stopping all volumes") # start the volume g.log.info("Starting volume %s", self.volname) ret = volume_start(self.mnode, self.volname) self.assertTrue(ret, ("Failed to start volume %s" % self.volname)) g.log.info("Volume %s started successfully", self.volname) # Verify volume's all process are online for 60 sec g.log.info("Verifying volume's all process are online") ret = wait_for_volume_process_to_be_online(self.mnode, self.volname, 60) self.assertTrue(ret, ("Volume %s : All process are not " "online", self.volname)) g.log.info("Successfully Verified volume %s processes are online", self.volname) # Verfiy glustershd process releases its parent process g.log.info("verifying self heal daemon process is daemonized") ret = is_shd_daemonized(nodes) self.assertTrue(ret, ("Either no self heal daemon process found or " "more than one self heal daemon process " "found : %s" % pids)) # check the self-heal daemon process g.log.info("Starting to get self-heal daemon process on " "nodes %s", nodes) ret, pids = get_self_heal_daemon_pid(nodes) self.assertTrue(ret, ("Either no self heal daemon process found or " "more than one self heal daemon process " "found : %s" % pids)) g.log.info( "Successful in getting single self heal daemon process" " on all nodes %s", nodes) # detach extra servers from the cluster g.log.info("peer detaching extra servers %s from cluster", self.extra_servers) ret = peer_detach_servers(self.mnode, self.extra_servers) self.assertTrue( ret, "Failed to peer detach extra servers : %s" % self.extra_servers) g.log.info("Peer detach success for %s ", self.extra_servers)
def test_snap_info_from_detached_node(self): # pylint: disable=too-many-statements """ Create a volume with single brick Create a snapshot Activate the snapshot created Enabled uss on the volume Validated snap info on all the nodes Peer detach one node Validate /var/lib/glusterd/snaps on the detached node Probe the detached node """ # Creating volume with single brick on one node servers_info_single_node = { self.servers[0]: self.all_servers_info[self.servers[0]] } bricks_list = form_bricks_list(self.mnode, self.volname, 1, self.servers[0], servers_info_single_node) ret, _, _ = volume_create(self.servers[0], self.volname, bricks_list) self.assertEqual(ret, 0, "Volume creation failed") g.log.info("Volume %s created successfully", self.volname) # Create a snapshot of the volume without volume start should fail self.snapname = "snap1" ret, _, _ = snap_create(self.mnode, self.volname, self.snapname, timestamp=False) self.assertNotEqual(ret, 0, "Snapshot created without starting the volume") g.log.info("Snapshot creation failed as expected") # Start the volume ret, _, _ = volume_start(self.mnode, self.volname) self.assertEqual(ret, 0, "Failed to start the volume %s" % self.volname) g.log.info("Volume start succeeded") # Create a snapshot of the volume after volume start ret, _, _ = snap_create(self.mnode, self.volname, self.snapname, timestamp=False) self.assertEqual( ret, 0, "Snapshot creation failed on the volume %s" % self.volname) g.log.info("Snapshot create succeeded") # Activate snapshot created ret, _, err = snap_activate(self.mnode, self.snapname) self.assertEqual( ret, 0, "Snapshot activate failed with following error %s" % (err)) g.log.info("Snapshot activated successfully") # Enable uss self.vol_options['features.uss'] = 'enable' ret = set_volume_options(self.mnode, self.volname, self.vol_options) self.assertTrue( ret, "gluster volume set %s features.uss " "enable failed" % self.volname) g.log.info("gluster volume set %s features.uss " "enable successfully", self.volname) # Validate files /var/lib/glusterd/snaps on all the servers is same self.pathname = "/var/lib/glusterd/snaps/%s" % self.snapname for server in self.servers: conn = g.rpyc_get_connection(server) ret = conn.modules.os.path.isdir(self.pathname) self.assertTrue( ret, "%s directory doesn't exist on node %s" % (self.pathname, server)) g.log.info("%s path exists on node %s", self.pathname, server) g.rpyc_close_deployed_servers() # Peer detach one node self.random_node_peer_detach = random.choice(self.servers[1:]) ret = peer_detach_servers(self.mnode, self.random_node_peer_detach, validate=True) self.assertTrue( ret, "Peer detach of node: %s failed" % self.random_node_peer_detach) g.log.info("Peer detach succeeded") # /var/lib/glusterd/snaps/<snapname> directory should not present conn = g.rpyc_get_connection(self.random_node_peer_detach) ret = conn.modules.os.path.isdir(self.pathname) self.assertFalse( ret, "%s directory should not exist on the peer" "which is detached from cluster%s" % (self.pathname, self.random_node_peer_detach)) g.log.info("Expected: %s path doesn't exist on peer detached node %s", self.pathname, self.random_node_peer_detach) g.rpyc_close_deployed_servers() # Peer probe the detached node ret, _, _ = peer_probe(self.mnode, self.random_node_peer_detach) self.assertEqual( ret, 0, "Peer probe of node: %s failed" % self.random_node_peer_detach) g.log.info("Peer probe succeeded") # Validating peers are in connected state count = 0 while count < 10: sleep(2) ret = self.validate_peers_are_connected() if ret: break count += 1 self.assertTrue(ret, "Peers are not in connected state") g.log.info("Peer are in connected state")
def test_glusterd_statedump_when_quorum_set_on_volumes(self): """ Test Case: 1. Create and start a volume 2. Enable quota on the volume 2. Fuse mount the volume 3. Get the glusterd statedump and analyze the statedump 4. Enable client-side quorum on the volume 5. Get the glusterd statedump 6. Delete the volume and peer detach 2 nodes 7. Create a replica 2 volume and start it 8. Kill the first brick of the volume 9. Get the glusterd statedump 10. Start the volume with force 11. Enable server-side quorum on the volume 12. Get the glusterd statedump 13. Stop glusterd on one of the node 14. Get the glusterd statedump 15. Stop glusterd on another node 16. Get the glusterd statedump """ # pylint: disable=too-many-statements # Enable Quota ret, _, _ = quota_enable(self.mnode, self.volname) self.assertEqual( ret, 0, ("Failed to enable quota on the volume %s", self.volname)) g.log.info("Successfully enabled quota on the volume %s", self.volname) # Mount the volume ret, _, _ = mount_volume(self.volname, mtype=self.mount_type, mpoint=self.mounts[0].mountpoint, mserver=self.mnode, mclient=self.mounts[0].client_system) self.assertEqual(ret, 0, "Failed to mount the volume: %s" % self.volname) g.log.info("Successfully mounted the volume: %s", self.volname) # Get the statedump of glusterd process self.dump_count = 1 self._get_statedump_of_glusterd(self.dump_count) # Analyze the statedump created self._analyze_statedump() # Enable client-side quorum on volume option = {"cluster.quorum-type": "auto"} self._set_option_for_volume(self.volname, option) # Get the statedump of glusterd process self.dump_count += 1 self._get_statedump_of_glusterd(self.dump_count) # Delete the volume ret = cleanup_volume(self.mnode, self.volname) self.assertTrue(ret, "Failed to delete the volume: %s" % self.volname) g.log.info("Successfully deleted the volume: %s", self.volname) # Peer detach two nodes ret = peer_detach_servers(self.mnode, self.servers[1:3]) self.assertTrue(ret, "Failed to detach the servers %s" % self.servers[1:3]) g.log.info("Successfully detached peers %s", self.servers[1:3]) # Create a new replica 2 volume in the updated cluster self.volume_config = { 'name': 'test_glusterd_statedump_when_quorum_' 'set_on_volumes_replica-volume', 'servers': self.servers[3:], 'voltype': { 'type': 'replicated', 'replica_count': 2 }, } # Create and start a volume ret = setup_volume(self.mnode, self.all_servers_info, self.volume_config) self.assertTrue( ret, "Failed to create and start the volume: %s" % self.volume_config['name']) g.log.info("Volume %s created and started successfully", self.volume_config['name']) # Get the list of bricks in volume all_bricks = get_all_bricks(self.mnode, self.volume_config['name']) self.assertIsNotNone(all_bricks, "Unable to get list of bricks") # Kill the first brick in volume ret = bring_bricks_offline(self.volume_config['name'], all_bricks[0]) self.assertTrue(ret, "Unable to bring brick %s offline" % all_bricks[0]) g.log.info("Successfully brought the brick %s offline ", all_bricks[0]) # Get the statedump of glusterd process self.dump_count += 1 self._get_statedump_of_glusterd(self.dump_count) # Start the volume with force ret, _, _ = volume_start(self.mnode, self.volume_config['name'], True) self.assertEqual( ret, 0, "Failed to start volume %s with force" % self.volume_config['name']) g.log.info("Successfully started volume %s with force", self.volume_config['name']) # Enable server-side quorum on volume option = {"cluster.server-quorum-type": "server"} self._set_option_for_volume(self.volume_config['name'], option) # Get the statedump of glusterd process self.dump_count += 1 self._get_statedump_of_glusterd(self.dump_count) # Stop glusterd process on one of the node. self._stop_gluster(self.servers[3]) # Get the statedump of glusterd process self.dump_count += 1 self._get_statedump_of_glusterd(self.dump_count) # Stop glusterd process on one of the node. self._stop_gluster(self.servers[4]) # Get the statedump of glusterd process self.dump_count += 1 self._get_statedump_of_glusterd(self.dump_count)