def cluster_server_groups_write(self,username,password,host,port=8091, servers=None,cluster=None,httpCode=None,user_role=None): _cluster_server_groups_write = { "create_server_group":"pools/default/serverGroups;POST", "delete_server_group":"pools/default/serverGroups/<uuid>;DELETE", "edit_server_group":"pools/default/serverGroups;PUT", "edit_server_group_id":"pools/default/serverGroups/<uuid>;PUT" } rest = RestConnection(servers[0]) try: rest.delete_zone('rbacNewGroup') rest.delete_zone('rbackGroup') rest.delete_zone('rbacNewGroupUpdated') create_server_group = {"create_server_group":"pools/default/serverGroups;POST;" + "{'name':'rbacGroup'}"} result = self._return_http_code(create_server_group,username,password,host=host,port=port, httpCode=httpCode, user_role=user_role) zones = rest.get_zone_names() create_server_group = {"delete_server_group":"pools/default/serverGroups/" + zones['rbacGroup'] + ";DELETE"} result = self._return_http_code(create_server_group,username,password,host=host,port=port, httpCode=httpCode, user_role=user_role) rest.add_zone('rbacNewGroup') zones = rest.get_zone_names() create_server_group = {"create_server_group":"pools/default/serverGroups/" + zones['rbacNewGroup'] + ";PUT;" + "{'name':'rbacNewGroupUpdated'}"} result = self._return_http_code(create_server_group,username,password,host=host,port=port, httpCode=httpCode, user_role=user_role) except: print "Issues with Server Group add test case"
def test_delete_empty_defautl_zone(self): zone_name = "test1" default_zone = "Group 1" moved_node = [] serverInfo = self.servers[0] moved_node.append(serverInfo.ip) rest = RestConnection(serverInfo) try: self.log.info("create zone {0}".format(zone_name)) rest.add_zone(zone_name) if rest.is_zone_exist(zone_name): self.log.info("Move node {0} from zone {1} to zone {2}" \ .format(moved_node, default_zone, zone_name)) status = rest.shuffle_nodes_in_zones(moved_node, default_zone, zone_name) if status: rest.delete_zone(default_zone) else: self.fail("Failed to move node {0} from zone {1} to zone {2}" \ .format(moved_node, default_zone, zone_name)) if not rest.is_zone_exist(default_zone): self.log.info("successful delete default zone") else: raise Exception("Failed to delete default zone") rest.rename_zone(zone_name, default_zone) except Exception as e: print(e)
def shuffle_nodes_between_two_zones(self): """ Creates 'Group 2' zone and shuffles nodes between Group 1 and Group 2 in an alternate manner ie; 1st node in Group 1, 2nd node in Group 2, 3rd node in Group 1 and so on and finally rebalances the resulting cluster :return: nodes of 2nd zone """ serverinfo = self.cluster.master rest = RestConnection(serverinfo) zones = ["Group 1", "Group 2"] rest.add_zone("Group 2") nodes_in_zone = {"Group 1": [serverinfo.ip], "Group 2": []} second_zone_servers = list() # Keep track of second zone's nodes # Divide the nodes between zones. for i in range(1, len(self.nodes_in_cluster)): server_group = i % 2 nodes_in_zone[zones[server_group]].append( self.nodes_in_cluster[i].ip) if zones[server_group] == "Group 2": second_zone_servers.append(self.nodes_in_cluster[i]) # Shuffle the nodes node_in_zone = list( set(nodes_in_zone[zones[1]]) - set([node for node in rest.get_nodes_in_zone(zones[1])])) rest.shuffle_nodes_in_zones(node_in_zone, zones[0], zones[1]) self.task.rebalance(self.nodes_in_cluster, [], []) return second_zone_servers
def test_delete_empty_defautl_zone(self): zone_name ="test1" default_zone = "Group 1" moved_node = [] serverInfo = self.servers[0] moved_node.append(serverInfo.ip) rest = RestConnection(serverInfo) try: self.log.info("create zone {0}".format(zone_name)) rest.add_zone(zone_name) if rest.is_zone_exist(zone_name): self.log.info("Move node {0} from zone {1} to zone {2}" \ .format(moved_node, default_zone, zone_name)) status = rest.shuffle_nodes_in_zones(moved_node, default_zone, zone_name) if status: rest.delete_zone(default_zone) else: self.fail("Failed to move node {0} from zone {1} to zone {2}" \ .format(moved_node, default_zone, zone_name)) if not rest.is_zone_exist(default_zone): self.log.info("successful delete default zone") else: raise Exception("Failed to delete default zone") rest.rename_zone(zone_name, default_zone) except Exception,e : print e
def shuffle_nodes_between_zones_and_rebalance(self, to_remove=None): """ Shuffle the nodes present in the cluster if zone > 1. Rebalance the nodes in the end. Nodes are divided into groups iteratively. i.e: 1st node in Group 1, 2nd in Group 2, 3rd in Group 1 & so on, when zone=2 :param to_remove: List of nodes to be removed. """ if not to_remove: to_remove = [] serverinfo = self.servers[0] rest = RestConnection(serverinfo) zones = ["Group 1"] nodes_in_zone = {"Group 1": [serverinfo.ip]} # Create zones, if not existing, based on params zone in test. # Shuffle the nodes between zones. if int(self.zone) > 1: for i in range(1, int(self.zone)): a = "Group " zones.append(a + str(i + 1)) if not rest.is_zone_exist(zones[i]): rest.add_zone(zones[i]) nodes_in_zone[zones[i]] = [] # Divide the nodes between zones. nodes_in_cluster = \ [node.ip for node in self.cluster_util.get_nodes_in_cluster( self.cluster)] nodes_to_remove = [node.ip for node in to_remove] for i in range(1, len(self.servers)): if self.servers[i].ip in nodes_in_cluster \ and self.servers[i].ip not in nodes_to_remove: server_group = i % int(self.zone) nodes_in_zone[zones[server_group]].append( self.servers[i].ip) # Shuffle the nodesS for i in range(1, self.zone): node_in_zone = list( set(nodes_in_zone[zones[i]]) - set([node for node in rest.get_nodes_in_zone(zones[i])])) rest.shuffle_nodes_in_zones(node_in_zone, zones[0], zones[i]) otpnodes = [node.id for node in rest.node_statuses()] nodes_to_remove = [ node.id for node in rest.node_statuses() if node.ip in [t.ip for t in to_remove] ] # Start rebalance and monitor it. started = rest.rebalance(otpNodes=otpnodes, ejectedNodes=nodes_to_remove) if started: result = rest.monitorRebalance() self.assertTrue(result, msg="Rebalance failed{}".format(result)) msg = "successfully rebalanced cluster {0}" self.log.info(msg.format(result)) # Verify replicas of one node should not be in the same zone # as active vbuckets of the node. if self.zone > 1: self.cluster_util.verify_replica_distribution_in_zones( self.cluster, nodes_in_zone)
def test_create_zone_with_lower_case_name(self): zone_name = "allwithlowercaseeeeeee" serverInfo = self.servers[0] rest = RestConnection(serverInfo) try: self.log.info("create zone {0}".format(zone_name)) rest.add_zone(zone_name) except Exception, e: print e
def test_create_zone_with_upper_case_name(self): zone_name = "ALLWITHUPTERCASE" serverInfo = self.servers[0] rest = RestConnection(serverInfo) try: self.log.info("create zone {0}".format(zone_name)) rest.add_zone(zone_name) except Exception, e: print e
def test_create_zone_with_all_number_name(self): zone_name = "3223345557666760" serverInfo = self.servers[0] rest = RestConnection(serverInfo) try: self.log.info("create zone {0}".format(zone_name)) rest.add_zone(zone_name) except Exception, e: print e
def test_create_zone_with_all_number_name(self): zone_name = "3223345557666760" serverInfo = self.servers[0] rest = RestConnection(serverInfo) try: self.log.info("create zone {0}".format(zone_name)) rest.add_zone(zone_name) except Exception,e : print e
def test_create_second_default_zone(self): zone_name = "Group 1" serverInfo = self.servers[0] rest = RestConnection(serverInfo) try: self.log.info("create additional default zone") rest.add_zone(zone_name) except Exception as e: print(e)
def test_create_zone_with_upper_lower_number_and_space_name(self): zone_name = " AAAB BBCCC aakkk kmmm3 456 72 " serverInfo = self.servers[0] rest = RestConnection(serverInfo) try: self.log.info("create zone {0}".format(zone_name)) rest.add_zone(zone_name) except Exception,e : print e
def test_create_second_default_zone(self): zone_name = "Group 1" serverInfo = self.servers[0] rest = RestConnection(serverInfo) try: self.log.info("create additional default zone") rest.add_zone(zone_name) except Exception,e : print e
def test_create_zone_with_upper_case_name(self): zone_name = "ALLWITHUPTERCASE" serverInfo = self.servers[0] rest = RestConnection(serverInfo) try: self.log.info("create zone {0}".format(zone_name)) rest.add_zone(zone_name) except Exception,e : print e
def test_create_zone_with_lower_case_name(self): zone_name = "allwithlowercaseeeeeee" serverInfo = self.servers[0] rest = RestConnection(serverInfo) try: self.log.info("create zone {0}".format(zone_name)) rest.add_zone(zone_name) except Exception,e : print e
def test_create_zone_with_upper_lower_number_and_space_name(self): zone_name = " AAAB BBCCC aakkk kmmm3 456 72 " serverInfo = self.cluster.servers[0] rest = RestConnection(serverInfo) try: self.log.info("create zone {0}".format(zone_name)) rest.add_zone(zone_name) except Exception, e: print e
def test_create_zone_with_none_ascii_name(self): # zone name is limited to 64 bytes zone_name = "abcdGHIJKLMNOPQRSTUVWXYZ0123456789efghijklmnopqrstuvwyABCDEF_-.%" serverInfo = self.servers[0] rest = RestConnection(serverInfo) try: self.log.info("create zone {0}".format(zone_name)) rest.add_zone(zone_name) except Exception, e: print e
def test_create_zone_with_none_ascii_name(self): # zone name is limited to 64 bytes zone_name = "abcdGHIJKLMNOPQRSTUVWXYZ0123456789efghijklmnopqrstuvwyABCDEF_-.%" serverInfo = self.servers[0] rest = RestConnection(serverInfo) try: self.log.info("create zone {0}".format(zone_name)) rest.add_zone(zone_name) except Exception,e : print e
def shuffle_nodes_between_zones_and_rebalance(self, to_remove=None): """ Shuffle the nodes present in the cluster if zone > 1. Rebalance the nodes in the end. Nodes are divided into groups iteratively i.e. 1st node in Group 1, 2nd in Group 2, 3rd in Group 1 and so on, when zone=2. :param to_remove: List of nodes to be removed. """ if not to_remove: to_remove = [] serverinfo = self.orchestrator rest = RestConnection(serverinfo) zones = ["Group 1"] nodes_in_zone = {"Group 1": [serverinfo.ip]} # Create zones, if not existing, based on params zone in test. # Shuffle the nodes between zones. if int(self.zone) > 1: for i in range(1, int(self.zone)): a = "Group " zones.append(a + str(i + 1)) if not rest.is_zone_exist(zones[i]): rest.add_zone(zones[i]) nodes_in_zone[zones[i]] = [] # Divide the nodes between zones. nodes_in_cluster = [ node.ip for node in self.get_nodes_in_cluster() ] nodes_to_remove = [node.ip for node in to_remove] for i in range(1, len(self.servers)): if self.servers[i].ip in nodes_in_cluster and self.servers[ i].ip not in nodes_to_remove: server_group = i % int(self.zone) nodes_in_zone[zones[server_group]].append( self.servers[i].ip) # Shuffle the nodesS for i in range(1, self.zone): node_in_zone = list( set(nodes_in_zone[zones[i]]) - {node for node in rest.get_nodes_in_zone(zones[i])}) rest.shuffle_nodes_in_zones(node_in_zone, zones[0], zones[i]) self.zones = nodes_in_zone otpnodes = [node.id for node in rest.node_statuses()] nodes_to_remove = [ node.id for node in rest.node_statuses() if node.ip in [t.ip for t in to_remove] ] # Start rebalance and monitor it. started = rest.rebalance(otpNodes=otpnodes, ejectedNodes=nodes_to_remove) if started: result = rest.monitorRebalance() msg = "successfully rebalanced cluster {0}" self.log.info(msg.format(result))
def test_zone_enable_after_upgrade_from_ce_to_ee(self): params = {} params['product'] = self.product params['version'] = self.version params['vbuckets'] = [self.vbuckets] params['type'] = self.type """ install couchbasse server community edition to run the test """ InstallerJob().parallel_install(self.servers[:3], params) params["type"] = "enterprise" zone_name = "AAABBBCCCaakkkkmmm345672" serverInfo = self.servers[0] ini_servers = self.servers[:self.nodes_init] rest = RestConnection(serverInfo) self.user = serverInfo.rest_username self.password = serverInfo.rest_password if len(ini_servers) > 1: self.cluster.rebalance([ini_servers[0]], ini_servers[1:], []) rest = RestConnection(self.master) self._bucket_creation() """ verify all nodes in cluster in CE """ if rest.is_enterprise_edition(): raise Exception("This test needs couchbase server community edition to run") self._load_all_buckets(self.servers[0], self.gen_load, "create", 0) try: self.log.info("create zone {0}".format(zone_name)) result = rest.add_zone(zone_name) if result: raise Exception("Zone feature should not be available in CE version") except Exception,e : if "Failed" in e: pass
def test_zone_enable_after_upgrade_from_ce_to_ee(self): params = {} params['product'] = self.product params['version'] = self.version params['vbuckets'] = [self.vbuckets] params['type'] = self.type """ install couchbasse server community edition to run the test """ InstallerJob().parallel_install(self.servers[:3], params) params["type"] = "enterprise" zone_name = "AAABBBCCCaakkkkmmm345672" serverInfo = self.servers[0] ini_servers = self.servers[:self.nodes_init] rest = RestConnection(serverInfo) self.user = serverInfo.rest_username self.password = serverInfo.rest_password if len(ini_servers) > 1: self.cluster.rebalance([ini_servers[0]], ini_servers[1:], []) rest = RestConnection(self.master) self._bucket_creation() """ verify all nodes in cluster in CE """ if rest.is_enterprise_edition(): raise Exception( "This test needs couchbase server community edition to run") self._load_all_buckets(self.servers[0], self.gen_load, "create", 0) try: self.log.info("create zone {0}".format(zone_name)) result = rest.add_zone(zone_name) if result: raise Exception( "Zone feature should not be available in CE version") except Exception, e: if "Failed" in e: pass
def shuffle_nodes_between_zones_and_rebalance(self, to_remove=None): """ Shuffle the nodes present in the cluster if zone > 1. Rebalance the nodes in the end. Nodes are divided into groups iteratively i.e. 1st node in Group 1, 2nd in Group 2, 3rd in Group 1 and so on, when zone=2. :param to_remove: List of nodes to be removed. """ if not to_remove: to_remove = [] serverinfo = self.servers[0] rest = RestConnection(serverinfo) zones = ["Group 1"] nodes_in_zone = {"Group 1": [serverinfo.ip]} # Create zones, if not existing, based on params zone in test. # Shuffle the nodes between zones. if int(self.zone) > 1: for i in range(1, int(self.zone)): a = "Group " zones.append(a + str(i + 1)) if not rest.is_zone_exist(zones[i]): rest.add_zone(zones[i]) nodes_in_zone[zones[i]] = [] # Divide the nodes between zones. nodes_in_cluster = [node.ip for node in self.get_nodes_in_cluster()] nodes_to_remove = [node.ip for node in to_remove] for i in range(1, len(self.servers)): if self.servers[i].ip in nodes_in_cluster and self.servers[i].ip not in nodes_to_remove: server_group = i % int(self.zone) nodes_in_zone[zones[server_group]].append(self.servers[i].ip) # Shuffle the nodesS for i in range(1, self.zone): node_in_zone = list(set(nodes_in_zone[zones[i]]) - set([node for node in rest.get_nodes_in_zone(zones[i])])) rest.shuffle_nodes_in_zones(node_in_zone, zones[0], zones[i]) otpnodes = [node.id for node in rest.node_statuses()] nodes_to_remove = [node.id for node in rest.node_statuses() if node.ip in [t.ip for t in to_remove]] # Start rebalance and monitor it. started = rest.rebalance(otpNodes=otpnodes, ejectedNodes=nodes_to_remove) if started: result = rest.monitorRebalance() msg = "successfully rebalanced cluster {0}" self.log.info(msg.format(result)) # Verify replicas of one node should not be in the same zone as active vbuckets of the node. if self.zone > 1: self._verify_replica_distribution_in_zones(nodes_in_zone)
if result: raise Exception(\ "Zone feature should not be available in CE version") except Exception, e: if "Failed" in e: pass serverInfo = self.servers[1] rest = RestConnection(serverInfo) self.user = serverInfo.rest_username self.password = serverInfo.rest_password if not rest.is_enterprise_edition(): raise Exception("Test failed to upgrade cluster from CE to EE") self.log.info("try to create zone {0} " "when cluster {1} is completely EE".format( zone_name, serverInfo.ip)) result = rest.add_zone(zone_name) self.log.info("sleep 5 seconds") time.sleep(5) if result: self.log.info("Zone feature is available in this cluster") else: raise Exception("Could not create zone with name: %s in cluster. " "It's a bug" % zone_name) if rest.is_zone_exist(zone_name.strip()): self.log.info("verified! zone '{0}' is existed".format( zone_name.strip())) else: raise Exception("There is not zone with name: %s in cluster. " "It's a bug" % zone_name) """ re-install enterprise edition for next test if there is any """ InstallerJob().parallel_install([self.servers[0]], params)
class CommunityTests(CommunityBaseTest): def setUp(self): super(CommunityTests, self).setUp() self.command = self.input.param("command", "") self.zone = self.input.param("zone", 1) self.replica = self.input.param("replica", 1) self.command_options = self.input.param("command_options", '') self.set_get_ratio = self.input.param("set_get_ratio", 0.9) self.item_size = self.input.param("item_size", 128) self.shutdown_zone = self.input.param("shutdown_zone", 1) self.do_verify = self.input.param("do-verify", True) self.num_node = self.input.param("num_node", 4) self.services = self.input.param("services", None) self.start_node_services = self.input.param("start_node_services", "kv") self.add_node_services = self.input.param("add_node_services", "kv") self.timeout = 6000 self.user_add = self.input.param("user_add", None) self.user_role = self.input.param("user_role", None) def tearDown(self): super(CommunityTests, self).tearDown() def test_disabled_zone(self): disabled_zone = False zone_name = "group1" serverInfo = self.servers[0] self.rest = RestConnection(serverInfo) try: self.log.info("create zone name 'group1'!") result = self.rest.add_zone(zone_name) print("result ", result) except Exception as e: if e: print(e) disabled_zone = True pass if not disabled_zone: self.fail("CE version should not have zone feature") def check_audit_available(self): audit_available = False try: self.rest.getAuditSettings() audit_available = True except Exception as e: if e: print(e) if audit_available: self.fail("This feature 'audit' only available on " "Enterprise Edition") def check_ldap_available(self): ldap_available = False self.rest = RestConnection(self.master) try: s, c, h = self.rest.clearLDAPSettings() if s: ldap_available = True except Exception as e: if e: print(e) if ldap_available: self.fail("This feature 'ldap' only available on " "Enterprise Edition") def check_set_services(self): self.rest.force_eject_node() self.sleep(7, "wait for node reset done") try: status = self.rest.init_node_services(hostname=self.master.ip, services=[self.services]) except Exception as e: if e: print(e) if self.services == "kv": if status: self.log.info("CE could set {0} only service.".format( self.services)) else: self.fail("Failed to set {0} only service.".format( self.services)) elif self.services == "index,kv": if status: self.fail("CE does not support kv and index on same node") else: self.log.info("services enforced in CE") elif self.services == "kv,n1ql": if status: self.fail("CE does not support kv and n1ql on same node") else: self.log.info("services enforced in CE") elif self.services == "kv,eventing": if status: self.fail("CE does not support kv and eventing on same node") else: self.log.info("services enforced in CE") elif self.services == "index,n1ql": if status: self.fail("CE does not support index and n1ql on same node") else: self.log.info("services enforced in CE") elif self.services == "index,kv,n1ql": if status: self.log.info( "CE could set all services {0} on same nodes.".format( self.services)) else: self.fail("Failed to set kv, index and query services on CE") elif self.version[:5] in COUCHBASE_FROM_WATSON: if self.version[: 5] in COUCHBASE_FROM_VULCAN and "eventing" in self.services: if status: self.fail("CE does not support eventing in vulcan") else: self.log.info("services enforced in CE") elif self.services == "fts,index,kv": if status: self.fail( "CE does not support fts, index and kv on same node") else: self.log.info("services enforced in CE") elif self.services == "fts,index,n1ql": if status: self.fail( "CE does not support fts, index and n1ql on same node") else: self.log.info("services enforced in CE") elif self.services == "fts,kv,n1ql": if status: self.fail( "CE does not support fts, kv and n1ql on same node") else: self.log.info("services enforced in CE") elif self.services == "fts,index,kv,n1ql": if status: self.log.info( "CE could set all services {0} on same nodes.".format( self.services)) else: self.fail("Failed to set " "fts, index, kv, and query services on CE") else: self.fail("some services don't support") def check_set_services_when_add_node(self): self.rest.force_eject_node() sherlock_services_in_ce = ["kv", "index,kv,n1ql"] watson_services_in_ce = ["kv", "index,kv,n1ql", "fts,index,kv,n1ql"] self.sleep(5, "wait for node reset done") try: self.log.info("Initialize node with services {0}".format( self.start_node_services)) status = self.rest.init_node_services( hostname=self.master.ip, services=[self.start_node_services]) init_node = self.cluster.async_init_node( self.master, services=[self.start_node_services]) except Exception as e: if e: print(e) if not status: if self.version not in COUCHBASE_FROM_WATSON and \ self.start_node_services not in sherlock_services_in_ce: self.log.info( "initial services setting enforced in Sherlock CE") elif self.version in COUCHBASE_FROM_WATSON and \ self.start_node_services not in watson_services_in_ce: self.log.info("initial services setting enforced in Watson CE") elif status and init_node.result() != 0: add_node = False try: self.log.info("node with services {0} try to add".format( self.add_node_services)) add_node = self.cluster.rebalance( self.servers[:2], self.servers[1:2], [], services=[self.add_node_services]) except Exception: pass if add_node: self.get_services_map() list_nodes = self.get_nodes_from_services_map( get_all_nodes=True) map = self.get_nodes_services() if map[self.master.ip] == self.start_node_services and \ map[self.servers[1].ip] == self.add_node_services: self.log.info( "services set correctly when node added & rebalance") else: self.fail("services set incorrectly when node added & rebalance. " "cluster expected services: {0}; set cluster services {1} ." "add node expected srv: {2}; set add node srv {3}"\ .format(map[self.master.ip], self.start_node_services, \ map[self.servers[1].ip], self.add_node_services)) else: if self.version not in COUCHBASE_FROM_WATSON: if self.start_node_services in ["kv", "index,kv,n1ql"] and \ self.add_node_services not in ["kv", "index,kv,n1ql"]: self.log.info("services are enforced in CE") elif self.start_node_services not in [ "kv", "index,kv,n1ql" ]: self.log.info("services are enforced in CE") else: self.fail("maybe bug in add node") elif self.version in COUCHBASE_FROM_WATSON: if self.start_node_services in ["kv", "index,kv,n1ql", "fts,index,kv,n1ql"] and self.add_node_services not in \ ["kv", "index,kv,n1ql", "fts,index,kv,n1ql"]: self.log.info("services are enforced in CE") elif self.start_node_services not in [ "kv", "index,kv,n1ql", "fts,index,kv,n1ql" ]: self.log.info("services are enforced in CE") else: self.fail("maybe bug in add node") else: self.fail("maybe bug in node initialization") def check_full_backup_only(self): """ for windows vm, ask IT to put uniq.exe at /cygdrive/c/Program Files (x86)/ICW/bin directory """ self.remote = RemoteMachineShellConnection(self.master) """ put params items=0 in test param so that init items = 0 """ self.remote.execute_command("{0}cbworkloadgen -n {1}:8091 -j -i 1000 " \ "-u Administrator -p password" \ .format(self.bin_path, self.master.ip)) """ delete backup location before run backup """ self.remote.execute_command("rm -rf {0}*".format(self.backup_location)) output, error = self.remote.execute_command("ls -lh {0}".format( self.backup_location)) self.remote.log_command_output(output, error) """ first full backup """ self.remote.execute_command("{0}cbbackup http://{1}:8091 {2} -m full " \ "-u Administrator -p password"\ .format(self.bin_path, self.master.ip, self.backup_c_location)) output, error = self.remote.execute_command("ls -lh {0}*/".format( self.backup_location)) self.remote.log_command_output(output, error) output, error = self.remote.execute_command("{0}cbtransfer -u Administrator "\ "-p password {1}*/*-full/ " \ "stdout: | grep set | uniq | wc -l"\ .format(self.bin_path, self.backup_c_location)) self.remote.log_command_output(output, error) if int(output[0]) != 1000: self.fail("full backup did not work in CE. " "Expected 1000, actual: {0}".format(output[0])) self.remote.execute_command("{0}cbworkloadgen -n {1}:8091 -j -i 1000 "\ " -u Administrator -p password --prefix=t_" .format(self.bin_path, self.master.ip)) """ do different backup mode """ self.remote.execute_command("{0}cbbackup -u Administrator -p password "\ "http://{1}:8091 {2} -m {3}"\ .format(self.bin_path, self.master.ip, self.backup_c_location, self.backup_option)) output, error = self.remote.execute_command("ls -lh {0}".format( self.backup_location)) self.remote.log_command_output(output, error) output, error = self.remote.execute_command("{0}cbtransfer -u Administrator "\ "-p password {1}*/*-{2}/ stdout: "\ "| grep set | uniq | wc -l"\ .format(self.bin_path, self.backup_c_location, self.backup_option)) self.remote.log_command_output(output, error) if int(output[0]) == 2000: self.log.info("backup option 'diff' is enforced in CE") elif int(output[0]) == 1000: self.fail("backup option 'diff' is not enforced in CE. " "Expected 2000, actual: {0}".format(output[0])) else: self.fail("backup failed to backup correct items") self.remote.disconnect() def check_ent_backup(self): """ for CE version from Watson, cbbackupmgr exe file should not in bin """ command = "cbbackupmgr" self.remote = RemoteMachineShellConnection(self.master) self.log.info("check if {0} in {1} directory".format( command, self.bin_path)) found = self.remote.file_exists(self.bin_path, command) if found: self.log.info("found {0} in {1} directory".format( command, self.bin_path)) self.fail("CE from Watson should not contain {0}".format(command)) elif not found: self.log.info("Ent. backup in CE is enforced, not in bin!") self.remote.disconnect() def check_memory_optimized_storage_mode(self): """ from Watson, CE should not have option 'memory_optimized' to set """ self.rest.force_eject_node() self.sleep(5, "wait for node reset done") try: self.log.info("Initialize node with 'Memory Optimized' option") status = self.rest.set_indexer_storage_mode( username=self.input.membase_settings.rest_username, password=self.input.membase_settings.rest_password, storageMode='memory_optimized') except Exception as ex: if ex: print(ex) if not status: self.log.info("Memory Optimized setting enforced in CE " "Could not set memory_optimized option") else: self.fail("Memory Optimzed setting does not enforced in CE " "We could set this option in") def check_x509_cert(self): """ from Watson, X509 certificate only support in EE """ api = self.rest.baseUrl + "pools/default/certificate?extended=true" self.log.info("request to get certificate at " "'pools/default/certificate?extended=true' " "should return False") try: status, content, header = self.rest._http_request(api, 'GET') except Exception as ex: if ex: print(ex) if status: self.fail("This X509 certificate feature only available in EE") elif not status: if "requires enterprise edition" in content: self.log.info("X509 cert is enforced in CE") def check_roles_base_access(self): """ from Watson, roles base access for admin should not in in CE """ if self.user_add is None: self.fail( "We need to pass user name (user_add) to run this test. ") if self.user_role is None: self.fail( "We need to pass user roles (user_role) to run this test. ") api = self.rest.baseUrl + "settings/rbac/users/" + self.user_add self.log.info("url to run this test: %s" % api) """ add admin user """ param = "name=%s&roles=%s" % (self.user_add, self.user_role) try: status, content, header = self.rest._http_request( api, 'PUT', param) except Exception as ex: if ex: print(ex) if status: self.fail("CE should not allow to add admin users") else: self.log.info("roles base is enforced in CE! ") def check_root_certificate(self): """ from watson, ce should not see root certificate manual test: curl -u Administrator:password -X GET http://localhost:8091/pools/default/certificate """ api = self.rest.baseUrl + "pools/default/certificate" try: status, content, header = self.rest._http_request(api, 'GET') except Exception as ex: if ex: print(ex) if status: self.fail("CE should not see root certificate!") elif "requires enterprise edition" in content: self.log.info("root certificate is enforced in CE! ") def check_settings_audit(self): """ from watson, ce should not set audit manual test: curl -u Administrator:password -X GET http://localhost:8091/settings/audit """ api = self.rest.baseUrl + "settings/audit" try: status, content, header = self.rest._http_request(api, 'GET') except Exception as ex: if ex: print(ex) if status: self.fail("CE should not allow to set audit !") elif "requires enterprise edition" in content: self.log.info("settings audit is enforced in CE! ") def check_infer(self): """ from watson, ce should not see infer manual test: curl -H "Content-Type: application/json" -X POST -d '{"statement":"infer `bucket_name`;"}' http://localhost:8093/query/service test params: new_services=kv-index-n1ql,default_bucket=False """ self.rest.force_eject_node() self.sleep(7, "wait for node reset done") self.rest.init_node() bucket = "default" self.rest.create_bucket(bucket, ramQuotaMB=200) api = self.rest.query_baseUrl + "query/service" param = urllib.parse.urlencode({"statement": "infer `%s` ;" % bucket}) try: status, content, header = self.rest._http_request( api, 'POST', param) json_parsed = json.loads(content) except Exception as ex: if ex: print(ex) if json_parsed["status"] == "success": self.fail("CE should not allow to run INFER !") elif json_parsed["status"] == "fatal": self.log.info("INFER is enforced in CE! ") def check_auto_complete(self): """ this feature has not complete to block in CE """ """ Check new features from spock start here """ def check_cbbackupmgr(self): """ cbbackupmgr should not available in CE from spock """ if self.cb_version[:5] in COUCHBASE_FROM_SPOCK: file_name = "cbbackupmgr" + self.file_extension self.log.info("check if cbbackupmgr in bin dir in CE") result = self.remote.file_exists(self.bin_path, file_name) if result: self.fail("cbbackupmgr should not in bin dir of CE") else: self.log.info("cbbackupmgr is enforced in CE") self.remote.disconnect() def test_max_ttl_bucket(self): """ From vulcan, EE bucket has has an option to set --max-ttl, not it CE. This test is make sure CE could not create bucket with option --max-ttl This test must pass default_bucket=False """ if self.cb_version[:5] not in COUCHBASE_FROM_VULCAN: self.log.info("This test only for vulcan and later") return cmd = 'curl -X POST -u Administrator:password \ http://{0}:8091/pools/default/buckets \ -d name=bucket0 \ -d maxTTL=100 \ -d authType=sasl \ -d ramQuotaMB=100 '.format(self.master.ip) if self.cli_test: cmd = "{0}couchbase-cli bucket-create -c {1}:8091 --username Administrator \ --password password --bucket bucket0 --bucket-type couchbase \ --bucket-ramsize 512 --bucket-replica 1 --bucket-priority high \ --bucket-eviction-policy fullEviction --enable-flush 0 \ --enable-index-replica 1 --max-ttl 200".format( self.bin_path, self.master.ip) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "Max TTL is supported in enterprise edition only" if self.cli_test: mesg = "Maximum TTL can only be configured on enterprise edition" if output and mesg not in str(output[0]): self.fail("max ttl feature should not in Community Edition") buckets = RestConnection(self.master).get_buckets() if buckets: for bucket in buckets: self.log.info("bucekt in cluser: {0}".format(bucket.name)) if bucket.name == "bucket0": self.fail("Failed to enforce feature max ttl in CE.") conn.disconnect() def test_setting_audit(self): """ CE does not allow to set audit from vulcan 5.5.0 """ if self.cb_version[:5] not in COUCHBASE_FROM_VULCAN: self.log.info("This test only for vulcan and later") return cmd = 'curl -X POST -u Administrator:password \ http://{0}:8091/settings/audit \ -d auditdEnabled=true '.format(self.master.ip) if self.cli_test: cmd = "{0}couchbase-cli setting-audit -c {1}:8091 -u Administrator \ -p password --audit-enabled 1 --audit-log-rotate-interval 604800 \ --audit-log-path /opt/couchbase/var/lib/couchbase/logs "\ .format(self.bin_path, self.master.ip) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "This http API endpoint requires enterprise edition" if output and mesg not in str(output[0]): self.fail("setting-audit feature should not in Community Edition") conn.disconnect() def test_setting_autofailover_enterprise_only(self): """ CE does not allow set auto failover if disk has issue and failover group from vulcan 5.5.0 """ if self.cb_version[:5] not in COUCHBASE_FROM_VULCAN: self.log.info("This test only for vulcan and later") return self.failover_disk_period = self.input.param("failover_disk_period", False) self.failover_server_group = self.input.param("failover_server_group", False) failover_disk_period = "" if self.failover_disk_period: if self.cli_test: failover_disk_period = "--failover-data-disk-period 300" else: failover_disk_period = "-d failoverOnDataDiskIssues[timePeriod]=300" failover_server_group = "" if self.failover_server_group and self.cli_test: failover_server_group = "--enable-failover-of-server-group 1" cmd = 'curl -X POST -u Administrator:password \ http://{0}:8091/settings/autoFailover -d enabled=true -d timeout=120 \ -d maxCount=1 \ -d failoverOnDataDiskIssues[enabled]=true {1} \ -d failoverServerGroup={2}'.format(self.master.ip, failover_disk_period, self.failover_server_group) if self.cli_test: cmd = "{0}couchbase-cli setting-autofailover -c {1}:8091 \ -u Administrator -p password \ --enable-failover-on-data-disk-issues 1 {2} {3} "\ .format(self.bin_path, self.master.ip, failover_disk_period, failover_server_group) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "Auto failover on Data Service disk issues can only be " + \ "configured on enterprise edition" if not self.cli_test: if self.failover_disk_period or \ self.failover_server_group: if output and not error: self.fail("setting autofailover disk issues feature\ should not in Community Edition") else: if self.failover_server_group: mesg = "--enable-failover-of-server-groups can only be " + \ "configured on enterprise edition" if output and mesg not in str(output[0]): self.fail("Setting EE autofailover features \ should not in Community Edition") else: self.log.info("EE setting autofailover are disable in CE") conn.disconnect() def test_set_bucket_compression(self): """ CE does not allow to set bucket compression to bucket from vulcan 5.5.0. Mode compression: off,active,passive Note: must set defaultbucket=False for this test """ if self.cb_version[:5] not in COUCHBASE_FROM_VULCAN: self.log.info("This test only for vulcan and later") return self.compression_mode = self.input.param("compression_mode", "off") cmd = 'curl -X POST -u Administrator:password \ http://{0}:8091/pools/default/buckets \ -d name=bucket0 \ -d compressionMode={1} \ -d authType=sasl \ -d ramQuotaMB=100 '.format( self.master.ip, self.compression_mode) if self.cli_test: cmd = "{0}couchbase-cli bucket-create -c {1}:8091 --username Administrator \ --password password --bucket bucket0 --bucket-type couchbase \ --bucket-ramsize 512 --bucket-replica 1 --bucket-priority high \ --bucket-eviction-policy fullEviction --enable-flush 0 \ --enable-index-replica 1 --compression-mode {2}".format( self.bin_path, self.master.ip, self.compression_mode) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "Compression mode is supported in enterprise edition only" if self.cli_test: mesg = "Compression mode can only be configured on enterprise edition" if output and mesg not in str(output[0]): self.fail("Setting bucket compression should not in CE") conn.disconnect()
def test_zone_enable_after_upgrade_from_ce_to_ee(self): self.services = self.input.param("services", "kv") params = {} params['product'] = self.product params['version'] = self.version params['vbuckets'] = [self.vbuckets] params['type'] = self.type """ install couchbasse server community edition to run the test """ InstallerJob().parallel_install(self.servers[:3], params) params["type"] = "enterprise" zone_name = "AAABBBCCCaakkkkmmm345672" serverInfo = self.servers[0] ini_servers = self.servers[:self.nodes_init] rest = RestConnection(serverInfo) self.user = serverInfo.rest_username self.password = serverInfo.rest_password if len(ini_servers) > 1: self.cluster.rebalance([ini_servers[0]], ini_servers[1:], [],\ services = self.services) rest = RestConnection(self.master) self._bucket_creation() """ verify all nodes in cluster in CE """ if rest.is_enterprise_edition(): raise Exception( "This test needs couchbase server community edition to run") self._load_all_buckets(self.servers[0], self.gen_load, "create", 0) try: self.log.info("create zone {0}".format(zone_name)) result = rest.add_zone(zone_name) if result: raise Exception( "Zone feature should not be available in CE version") except Exception as e: if "Failed" in e: pass for i in range(1, int(self.nodes_init) + 1): if i == 1: """ install EE on one node to do swap rebalance """ InstallerJob().parallel_install(self.servers[3:], params) self.cluster.rebalance([ini_servers[0]], \ [self.servers[int(self.nodes_init)]],\ [self.servers[int(self.nodes_init) - i]], services = self.services) self.log.info("sleep 5 seconds") time.sleep(5) try: self.log.info( "try to create zone {0} " "when cluster is not completely EE".format(zone_name)) result = rest.add_zone(zone_name) if result: raise Exception(\ "Zone feature should not be available in CE version") except Exception as e: if "Failed" in e: pass else: InstallerJob().parallel_install([self.servers[int(self.nodes_init)\ - (i - 1)]], params) self.cluster.rebalance([ini_servers[0]],\ [self.servers[int(self.nodes_init) - (i -1)]], [self.servers[int(self.nodes_init) - i]], services = self.services) self.sleep(12, "wait 12 seconds after rebalance") if i < int(self.nodes_init): try: self.log.info( "try to create zone {0} " "when cluster is not completely EE".format( zone_name)) result = rest.add_zone(zone_name) if result: raise Exception(\ "Zone feature should not be available in CE version") except Exception as e: if "Failed" in e: pass serverInfo = self.servers[1] rest = RestConnection(serverInfo) self.user = serverInfo.rest_username self.password = serverInfo.rest_password if not rest.is_enterprise_edition(): raise Exception("Test failed to upgrade cluster from CE to EE") self.log.info("try to create zone {0} " "when cluster {1} is completely EE".format( zone_name, serverInfo.ip)) result = rest.add_zone(zone_name) self.log.info("sleep 5 seconds") time.sleep(5) if result: self.log.info("Zone feature is available in this cluster") else: raise Exception("Could not create zone with name: %s in cluster. " "It's a bug" % zone_name) if rest.is_zone_exist(zone_name.strip()): self.log.info("verified! zone '{0}' is existed".format( zone_name.strip())) else: raise Exception("There is not zone with name: %s in cluster. " "It's a bug" % zone_name) """ re-install enterprise edition for next test if there is any """ InstallerJob().parallel_install([self.servers[0]], params) """ reset master node to new node to teardown cluster """ self.log.info("Start to clean up cluster") self.master = self.servers[1] self.servers = self.servers[1:]
def test_replica_distribution_in_zone(self): if len(self.servers) < int(self.num_node): msg = "This test needs minimum {1} servers to run.\n Currently in ini file \ has only {0} servers".format(len(self.servers), self.num_node) self.log.error("{0}".format(msg)) raise Exception(msg) if self.shutdown_zone >= self.zone: msg = "shutdown zone should smaller than zone" raise Exception(msg) serverInfo = self.servers[0] rest = RestConnection(serverInfo) zones = [] zones.append("Group 1") nodes_in_zone = {} nodes_in_zone["Group 1"] = [serverInfo.ip] """ Create zone base on params zone in test""" try: if int(self.zone) > 1: for i in range(1, int(self.zone)): a = "Group " zone_name = a + str(i + 1) zones.append(zone_name) rest.add_zone(zone_name) servers_rebalanced = [] self.user = serverInfo.rest_username self.password = serverInfo.rest_password if len(self.servers) % int(self.zone) != 0: msg = "unbalance zone. Recaculate to make balance ratio node/zone" raise Exception(msg) """ Add node to each zone """ k = 1 for i in range(0, self.zone): if "Group 1" in zones[i]: total_node_per_zone = int(len(self.servers)) // int( self.zone) - 1 else: nodes_in_zone[zones[i]] = [] total_node_per_zone = int(len(self.servers)) // int( self.zone) for n in range(0, total_node_per_zone): nodes_in_zone[zones[i]].append(self.servers[k].ip) rest.add_node(user=self.user, password=self.password, \ remoteIp=self.servers[k].ip, port='8091', zone_name=zones[i]) k += 1 otpNodes = [node.id for node in rest.node_statuses()] """ Start rebalance and monitor it. """ started = rest.rebalance(otpNodes, []) if started: try: result = rest.monitorRebalance() except RebalanceFailedException as e: self.log.error("rebalance failed: {0}".format(e)) return False, servers_rebalanced msg = "successfully rebalanced cluster {0}" self.log.info(msg.format(result)) """ Verify replica of one node should not in same zone of active. """ self._verify_replica_distribution_in_zones(nodes_in_zone, "tap") """ Simulate entire nodes down in zone(s) by killing erlang process""" shutdown_nodes = [] if self.shutdown_zone >= 1 and self.zone >= 2: self.log.info("Start to shutdown nodes in zone to failover") for down_zone in range(1, self.shutdown_zone + 1): down_zone = "Group " + str(down_zone + 1) for sv in nodes_in_zone[down_zone]: for si in self.servers: if si.ip == sv: server = si shutdown_nodes.append(si) shell = RemoteMachineShellConnection(server) shell.kill_erlang(self.os_name) """ Failover down node(s)""" self.log.info("----> start failover node %s" % server.ip) failed_over = rest.fail_over("ns_1@" + server.ip) if not failed_over: self.log.info( "unable to failover the node the first time. \ try again in 75 seconds..") time.sleep(75) failed_over = rest.fail_over("ns_1@" + server.ip) self.assertTrue( failed_over, "unable to failover node after erlang was killed") otpNodes = [node.id for node in rest.node_statuses()] self.log.info("----> start rebalance after failover.") """ Start rebalance and monitor it. """ started = rest.rebalance(otpNodes, []) if started: try: result = rest.monitorRebalance() except RebalanceFailedException as e: self.log.error("rebalance failed: {0}".format(e)) return False, servers_rebalanced msg = "successfully rebalanced in selected nodes from the cluster ? {0}" self.log.info(msg.format(result)) """ Compare current keys in bucekt with initial loaded keys count. """ self._verify_total_keys(self.servers[0], self.num_items) except Exception as e: self.log.error(e) raise finally: self.log.info("---> remove all nodes in all zones") if shutdown_nodes: for node in shutdown_nodes: conn = RemoteMachineShellConnection(node) self.log.info( "---> re-start nodes %s after erlang killed " % node.ip) conn.start_couchbase() time.sleep(5) BucketOperationHelper.delete_all_buckets_or_assert( self.servers, self) ClusterOperationHelper.cleanup_cluster(self.servers, master=self.master) self.log.info("---> remove all zones in cluster") rm_zones = rest.get_zone_names() for zone in rm_zones: if zone != "Group 1": rest.delete_zone(zone)
class AutoRetryFailedRebalance(RebalanceBaseTest): def setUp(self): super(AutoRetryFailedRebalance, self).setUp() self.rest = RestConnection(self.servers[0]) self.sleep_time = self.input.param("sleep_time", 15) self.enabled = self.input.param("enabled", True) self.afterTimePeriod = self.input.param("afterTimePeriod", 300) self.maxAttempts = self.input.param("maxAttempts", 1) self.log.info("Changing the retry rebalance settings ....") self.change_retry_rebalance_settings( enabled=self.enabled, afterTimePeriod=self.afterTimePeriod, maxAttempts=self.maxAttempts) self.rebalance_operation = self.input.param("rebalance_operation", "rebalance_out") self.disable_auto_failover = self.input.param("disable_auto_failover", True) self.auto_failover_timeout = self.input.param("auto_failover_timeout", 120) if self.disable_auto_failover: self.rest.update_autofailover_settings(False, 120) else: self.rest.update_autofailover_settings(True, self.auto_failover_timeout) def tearDown(self): self.reset_retry_rebalance_settings() # Reset to default value super(AutoRetryFailedRebalance, self).tearDown() rest = RestConnection(self.servers[0]) zones = rest.get_zone_names() for zone in zones: if zone != "Group 1": rest.delete_zone(zone) def test_auto_retry_of_failed_rebalance_where_failure_happens_before_rebalance( self): before_rebalance_failure = self.input.param("before_rebalance_failure", "stop_server") # induce the failure before the rebalance starts self._induce_error(before_rebalance_failure) self.sleep(self.sleep_time) try: operation = self._rebalance_operation(self.rebalance_operation) operation.result() except Exception as e: self.log.info("Rebalance failed with : {0}".format(str(e))) # Recover from the error self._recover_from_error(before_rebalance_failure) self.check_retry_rebalance_succeeded() else: self.fail( "Rebalance did not fail as expected. Hence could not validate auto-retry feature.." ) finally: if self.disable_auto_failover: self.rest.update_autofailover_settings(True, 120) self.start_server(self.servers[1]) self.stop_firewall_on_node(self.servers[1]) def test_auto_retry_of_failed_rebalance_where_failure_happens_during_rebalance( self): during_rebalance_failure = self.input.param("during_rebalance_failure", "stop_server") try: operation = self._rebalance_operation(self.rebalance_operation) self.sleep(self.sleep_time) # induce the failure during the rebalance self._induce_error(during_rebalance_failure) operation.result() except Exception as e: self.log.info("Rebalance failed with : {0}".format(str(e))) # Recover from the error self._recover_from_error(during_rebalance_failure) self.check_retry_rebalance_succeeded() else: # This is added as the failover task is not throwing exception if self.rebalance_operation == "graceful_failover": # Recover from the error self._recover_from_error(during_rebalance_failure) self.check_retry_rebalance_succeeded() else: self.fail( "Rebalance did not fail as expected. Hence could not validate auto-retry feature.." ) finally: if self.disable_auto_failover: self.rest.update_autofailover_settings(True, 120) self.start_server(self.servers[1]) self.stop_firewall_on_node(self.servers[1]) def test_auto_retry_of_failed_rebalance_does_not_get_triggered_when_rebalance_is_stopped( self): operation = self._rebalance_operation(self.rebalance_operation) reached = RestHelper(self.rest).rebalance_reached(30) self.assertTrue(reached, "Rebalance failed or did not reach {0}%".format(30)) self.rest.stop_rebalance(wait_timeout=self.sleep_time) result = json.loads(self.rest.get_pending_rebalance_info()) self.log.info(result) retry_rebalance = result["retry_rebalance"] if retry_rebalance != "not_pending": self.fail( "Auto-retry succeeded even when Rebalance was stopped by user") def test_negative_auto_retry_of_failed_rebalance_where_rebalance_will_be_cancelled( self): during_rebalance_failure = self.input.param("during_rebalance_failure", "stop_server") post_failure_operation = self.input.param("post_failure_operation", "cancel_pending_rebalance") try: operation = self._rebalance_operation(self.rebalance_operation) self.sleep(self.sleep_time) # induce the failure during the rebalance self._induce_error(during_rebalance_failure) operation.result() except Exception as e: self.log.info("Rebalance failed with : {0}".format(str(e))) # Recover from the error self._recover_from_error(during_rebalance_failure) result = json.loads(self.rest.get_pending_rebalance_info()) self.log.info(result) retry_rebalance = result["retry_rebalance"] rebalance_id = result["rebalance_id"] if retry_rebalance != "pending": self.fail("Auto-retry of failed rebalance is not triggered") if post_failure_operation == "cancel_pending_rebalance": # cancel pending rebalance self.log.info( "Cancelling rebalance Id: {0}".format(rebalance_id)) self.rest.cancel_pending_rebalance(rebalance_id) elif post_failure_operation == "disable_auto_retry": # disable the auto retry of the failed rebalance self.log.info( "Disable the the auto retry of the failed rebalance") self.change_retry_rebalance_settings(enabled=False) elif post_failure_operation == "retry_failed_rebalance_manually": # retry failed rebalance manually self.log.info( "Retrying failed rebalance Id: {0}".format(rebalance_id)) self.cluster.rebalance(self.servers[:self.nodes_init], [], []) else: self.fail("Invalid post_failure_operation option") # Now check and ensure retry won't happen result = json.loads(self.rest.get_pending_rebalance_info()) self.log.info(result) retry_rebalance = result["retry_rebalance"] if retry_rebalance != "not_pending": self.fail("Auto-retry of failed rebalance is not cancelled") else: self.fail( "Rebalance did not fail as expected. Hence could not validate auto-retry feature.." ) finally: if self.disable_auto_failover: self.rest.update_autofailover_settings(True, 120) self.start_server(self.servers[1]) self.stop_firewall_on_node(self.servers[1]) def test_negative_auto_retry_of_failed_rebalance_where_rebalance_will_not_be_cancelled( self): during_rebalance_failure = self.input.param("during_rebalance_failure", "stop_server") post_failure_operation = self.input.param("post_failure_operation", "create_delete_buckets") zone_name = "Group_{0}_{1}".format(random.randint(1, 1000000000), self._testMethodName) zone_name = zone_name[0:60] default_zone = "Group 1" moved_node = [] moved_node.append(self.servers[1].ip) try: operation = self._rebalance_operation(self.rebalance_operation) self.sleep(self.sleep_time) # induce the failure during the rebalance self._induce_error(during_rebalance_failure) operation.result() except Exception as e: self.log.info("Rebalance failed with : {0}".format(str(e))) # Recover from the error self._recover_from_error(during_rebalance_failure) result = json.loads(self.rest.get_pending_rebalance_info()) self.log.info(result) retry_rebalance = result["retry_rebalance"] if retry_rebalance != "pending": self.fail("Auto-retry of failed rebalance is not triggered") if post_failure_operation == "create_delete_buckets": # delete buckets and create new one BucketOperationHelper.delete_all_buckets_or_assert( servers=self.servers, test_case=self) self.sleep(self.sleep_time) BucketOperationHelper.create_bucket(self.master, test_case=self) elif post_failure_operation == "change_replica_count": # change replica count self.log.info("Changing replica count of buckets") for bucket in self.buckets: self.rest.change_bucket_props(bucket, replicaNumber=2) elif post_failure_operation == "change_server_group": # change server group self.log.info("Creating new zone " + zone_name) self.rest.add_zone(zone_name) self.log.info("Moving {0} to new zone {1}".format( moved_node, zone_name)) status = self.rest.shuffle_nodes_in_zones( moved_node, default_zone, zone_name) else: self.fail("Invalid post_failure_operation option") # In these failure scenarios while the retry is pending, then the retry will be attempted but fail try: self.check_retry_rebalance_succeeded() except Exception as e: self.log.info(e) if "Retrying of rebalance still did not help. All the retries exhausted" not in str( e): self.fail( "Auto retry of failed rebalance succeeded when it was expected to fail" ) else: self.fail( "Rebalance did not fail as expected. Hence could not validate auto-retry feature.." ) finally: if post_failure_operation == "change_server_group": status = self.rest.shuffle_nodes_in_zones( moved_node, zone_name, default_zone) self.log.info( "Shuffle the node back to default group . Status : {0}". format(status)) self.sleep(self.sleep_time) self.log.info("Deleting new zone " + zone_name) try: self.rest.delete_zone(zone_name) except: self.log.info("Errors in deleting zone") if self.disable_auto_failover: self.rest.update_autofailover_settings(True, 120) self.start_server(self.servers[1]) self.stop_firewall_on_node(self.servers[1]) def test_auto_retry_of_failed_rebalance_with_rebalance_test_conditions( self): test_failure_condition = self.input.param("test_failure_condition") # induce the failure before the rebalance starts self._induce_rebalance_test_condition(test_failure_condition) self.sleep(self.sleep_time) try: operation = self._rebalance_operation(self.rebalance_operation) operation.result() except Exception as e: self.log.info("Rebalance failed with : {0}".format(str(e))) # Delete the rebalance test condition so that we recover from the error self._delete_rebalance_test_condition(test_failure_condition) self.check_retry_rebalance_succeeded() else: self.fail( "Rebalance did not fail as expected. Hence could not validate auto-retry feature.." ) finally: if self.disable_auto_failover: self.rest.update_autofailover_settings(True, 120) self._delete_rebalance_test_condition(test_failure_condition) def test_auto_retry_of_failed_rebalance_with_autofailvoer_enabled(self): before_rebalance_failure = self.input.param("before_rebalance_failure", "stop_server") # induce the failure before the rebalance starts self._induce_error(before_rebalance_failure) try: operation = self._rebalance_operation(self.rebalance_operation) operation.result() except Exception as e: self.log.info("Rebalance failed with : {0}".format(str(e))) if self.auto_failover_timeout < self.afterTimePeriod: self.sleep(self.auto_failover_timeout) result = json.loads(self.rest.get_pending_rebalance_info()) self.log.info(result) retry_rebalance = result["retry_rebalance"] if retry_rebalance != "not_pending": self.fail( "Auto-failover did not cancel pending retry of the failed rebalance" ) else: try: self.check_retry_rebalance_succeeded() except Exception as e: if "Retrying of rebalance still did not help" not in str( e): self.fail( "retry rebalance succeeded even without failover") self.sleep(self.auto_failover_timeout) self.cluster.rebalance(self.servers[:self.nodes_init], [], []) else: self.fail( "Rebalance did not fail as expected. Hence could not validate auto-retry feature.." ) finally: if self.disable_auto_failover: self.rest.update_autofailover_settings(True, 120) self.start_server(self.servers[1]) self.stop_firewall_on_node(self.servers[1]) def _rebalance_operation(self, rebalance_operation): self.log.info("Starting rebalance operation of type : {0}".format( rebalance_operation)) if rebalance_operation == "rebalance_out": operation = self.cluster.async_rebalance( self.servers[:self.nodes_init], [], self.servers[1:]) elif rebalance_operation == "rebalance_in": operation = self.cluster.async_rebalance( self.servers[:self.nodes_init], [self.servers[self.nodes_init]], []) elif rebalance_operation == "swap_rebalance": self.rest.add_node(self.master.rest_username, self.master.rest_password, self.servers[self.nodes_init].ip, self.servers[self.nodes_init].port) operation = self.cluster.async_rebalance( self.servers[:self.nodes_init], [], [self.servers[self.nodes_init - 1]]) elif rebalance_operation == "graceful_failover": # TODO : retry for graceful failover is not yet implemented operation = self.cluster.async_failover( [self.master], failover_nodes=[self.servers[1]], graceful=True, wait_for_pending=120) return operation def _induce_error(self, error_condition): if error_condition == "stop_server": self.stop_server(self.servers[1]) elif error_condition == "enable_firewall": self.start_firewall_on_node(self.servers[1]) elif error_condition == "kill_memcached": self.kill_server_memcached(self.servers[1]) elif error_condition == "reboot_server": shell = RemoteMachineShellConnection(self.servers[1]) shell.reboot_node() elif error_condition == "kill_erlang": shell = RemoteMachineShellConnection(self.servers[1]) shell.kill_erlang() self.sleep(self.sleep_time * 3) else: self.fail("Invalid error induce option") def _recover_from_error(self, error_condition): if error_condition == "stop_server" or error_condition == "kill_erlang": self.start_server(self.servers[1]) self.sleep(self.sleep_time * 4) elif error_condition == "enable_firewall": self.stop_firewall_on_node(self.servers[1]) elif error_condition == "reboot_server": self.sleep(self.sleep_time * 4) # wait till node is ready after warmup ClusterOperationHelper.wait_for_ns_servers_or_assert( [self.servers[1]], self, wait_if_warmup=True) def _induce_rebalance_test_condition(self, test_failure_condition): if test_failure_condition == "verify_replication": set_command = "testconditions:set(verify_replication, {fail, \"" + "default" + "\"})" elif test_failure_condition == "backfill_done": set_command = "testconditions:set(backfill_done, {for_vb_move, \"" + "default\", 1 , " + "fail})" else: set_command = "testconditions:set({0}, fail)".format( test_failure_condition) get_command = "testconditions:get({0})".format(test_failure_condition) for server in self.servers: rest = RestConnection(server) _, content = rest.diag_eval(set_command) self.log.info("Command : {0} Return : {1}".format( set_command, content)) for server in self.servers: rest = RestConnection(server) _, content = rest.diag_eval(get_command) self.log.info("Command : {0} Return : {1}".format( get_command, content)) def _delete_rebalance_test_condition(self, test_failure_condition): delete_command = "testconditions:delete({0})".format( test_failure_condition) get_command = "testconditions:get({0})".format(test_failure_condition) for server in self.servers: rest = RestConnection(server) _, content = rest.diag_eval(delete_command) self.log.info("Command : {0} Return : {1}".format( delete_command, content)) for server in self.servers: rest = RestConnection(server) _, content = rest.diag_eval(get_command) self.log.info("Command : {0} Return : {1}".format( get_command, content))
def test_replica_distribution_in_zone(self): if len(self.servers) < int(self.num_node): msg = "This test needs minimum {1} servers to run.\n Currently in ini file \ has only {0} servers".format(len(self.servers), self.num_node) self.log.error("{0}".format(msg)) raise Exception(msg) if self.shutdown_zone >= self.zone: msg = "shutdown zone should smaller than zone" raise Exception(msg) serverInfo = self.servers[0] rest = RestConnection(serverInfo) zones = [] zones.append("Group 1") nodes_in_zone = {} nodes_in_zone["Group 1"] = [serverInfo.ip] """ Create zone base on params zone in test""" if int(self.zone) > 1: for i in range(1,int(self.zone)): a = "Group " zones.append(a + str(i + 1)) rest.add_zone(a + str(i + 1)) servers_rebalanced = [] self.user = serverInfo.rest_username self.password = serverInfo.rest_password if len(self.servers)%int(self.zone) != 0: msg = "unbalance zone. Recaculate to make balance ratio node/zone" raise Exception(msg) """ Add node to each zone """ k = 1 for i in range(0, self.zone): if "Group 1" in zones[i]: total_node_per_zone = int(len(self.servers))/int(self.zone) - 1 else: nodes_in_zone[zones[i]] = [] total_node_per_zone = int(len(self.servers))/int(self.zone) for n in range(0,total_node_per_zone): nodes_in_zone[zones[i]].append(self.servers[k].ip) rest.add_node(user=self.user, password=self.password, \ remoteIp=self.servers[k].ip, port='8091', zone_name=zones[i]) k += 1 otpNodes = [node.id for node in rest.node_statuses()] """ Start rebalance and monitor it. """ started = rest.rebalance(otpNodes, []) if started: try: result = rest.monitorRebalance() except RebalanceFailedException as e: self.log.error("rebalance failed: {0}".format(e)) return False, servers_rebalanced msg = "successfully rebalanced cluster {0}" self.log.info(msg.format(result)) """ Verify replica of one node should not in same zone of active. """ self._verify_replica_distribution_in_zones(nodes_in_zone, "tap") """ Simulate entire nodes down in zone(s) by killing erlang process""" if self.shutdown_zone >= 1 and self.zone >=2: self.log.info("Start to shutdown nodes in zone to failover") for down_zone in range(1, self.shutdown_zone + 1): down_zone = "Group " + str(down_zone + 1) for sv in nodes_in_zone[down_zone]: for si in self.servers: if si.ip == sv: server = si shell = RemoteMachineShellConnection(server) os_info = shell.extract_remote_info() shell.kill_erlang(os_info) """ Failover down node(s)""" failed_over = rest.fail_over("ns_1@" + server.ip) if not failed_over: self.log.info("unable to failover the node the first time. \ try again in 75 seconds..") time.sleep(75) failed_over = rest.fail_over("ns_1@" + server.ip) self.assertTrue(failed_over, "unable to failover node after erlang killed") otpNodes = [node.id for node in rest.node_statuses()] self.log.info("start rebalance after failover.") """ Start rebalance and monitor it. """ started = rest.rebalance(otpNodes, []) if started: try: result = rest.monitorRebalance() except RebalanceFailedException as e: self.log.error("rebalance failed: {0}".format(e)) return False, servers_rebalanced msg = "successfully rebalanced in selected nodes from the cluster ? {0}" self.log.info(msg.format(result)) """ Compare current keys in bucekt with initial loaded keys count. """ self._verify_total_keys(self.servers[0], self.num_items)
class CommunityTests(CommunityBaseTest): def setUp(self): super(CommunityTests, self).setUp() self.command = self.input.param("command", "") self.zone = self.input.param("zone", 1) self.replica = self.input.param("replica", 1) self.command_options = self.input.param("command_options", '') self.set_get_ratio = self.input.param("set_get_ratio", 0.9) self.item_size = self.input.param("item_size", 128) self.shutdown_zone = self.input.param("shutdown_zone", 1) self.do_verify = self.input.param("do-verify", True) self.num_node = self.input.param("num_node", 4) self.services = self.input.param("services", None) self.start_node_services = self.input.param("start_node_services", "kv") self.add_node_services = self.input.param("add_node_services", "kv") self.timeout = 6000 self.user_add = self.input.param("user_add", None) self.user_role = self.input.param("user_role", None) def tearDown(self): super(CommunityTests, self).tearDown() def test_disabled_zone(self): disabled_zone = False zone_name = "group1" serverInfo = self.servers[0] self.rest = RestConnection(serverInfo) try: self.log.info("create zone name 'group1'!") result = self.rest.add_zone(zone_name) print("result ", result) except Exception as e: if e: print(e) disabled_zone = True pass if not disabled_zone: self.fail("CE version should not have zone feature") def check_audit_available(self): audit_available = False try: self.rest.getAuditSettings() audit_available = True except Exception as e: if e: print(e) if audit_available: self.fail("This feature 'audit' only available on " "Enterprise Edition") def check_ldap_available(self): ldap_available = False self.rest = RestConnection(self.master) try: s, c, h = self.rest.clearLDAPSettings() if s: ldap_available = True except Exception as e: if e: print(e) if ldap_available: self.fail("This feature 'ldap' only available on " "Enterprise Edition") def check_set_services(self): self.rest.force_eject_node() self.sleep(7, "wait for node reset done") try: status = self.rest.init_node_services(hostname=self.master.ip, services=[self.services]) except Exception as e: if e: print(e) if self.services == "kv": if status: self.log.info("CE could set {0} only service.".format( self.services)) else: self.fail("Failed to set {0} only service.".format( self.services)) elif self.services == "index,kv": if status: self.fail("CE does not support kv and index on same node") else: self.log.info("services enforced in CE") elif self.services == "kv,n1ql": if status: self.fail("CE does not support kv and n1ql on same node") else: self.log.info("services enforced in CE") elif self.services == "kv,eventing": if status: self.fail("CE does not support kv and eventing on same node") else: self.log.info("services enforced in CE") elif self.services == "index,n1ql": if status: self.fail("CE does not support index and n1ql on same node") else: self.log.info("services enforced in CE") elif self.services == "index,kv,n1ql": if status: self.log.info( "CE could set all services {0} on same nodes.".format( self.services)) else: self.fail("Failed to set kv, index and query services on CE") elif self.version[:5] in COUCHBASE_FROM_WATSON: if self.version[: 5] in COUCHBASE_FROM_VULCAN and "eventing" in self.services: if status: self.fail("CE does not support eventing in vulcan") else: self.log.info("services enforced in CE") elif self.services == "fts,index,kv": if status: self.fail( "CE does not support fts, index and kv on same node") else: self.log.info("services enforced in CE") elif self.services == "fts,index,n1ql": if status: self.fail( "CE does not support fts, index and n1ql on same node") else: self.log.info("services enforced in CE") elif self.services == "fts,kv,n1ql": if status: self.fail( "CE does not support fts, kv and n1ql on same node") else: self.log.info("services enforced in CE") elif self.services == "fts,index,kv,n1ql": if status: self.log.info( "CE could set all services {0} on same nodes.".format( self.services)) else: self.fail("Failed to set " "fts, index, kv, and query services on CE") else: self.fail("some services don't support") def check_set_services_when_add_node(self): self.rest.force_eject_node() sherlock_services_in_ce = ["kv", "index,kv,n1ql"] watson_services_in_ce = ["kv", "index,kv,n1ql", "fts,index,kv,n1ql"] self.sleep(5, "wait for node reset done") kv_quota = 0 while kv_quota == 0: time.sleep(1) kv_quota = int(self.rest.get_nodes_self().mcdMemoryReserved) info = self.rest.get_nodes_self() kv_quota = int(info.mcdMemoryReserved * (CLUSTER_QUOTA_RATIO)) self.rest.set_service_memoryQuota(service='indexMemoryQuota', memoryQuota=INDEX_QUOTA) self.rest.set_service_memoryQuota(service='ftsMemoryQuota', memoryQuota=FTS_QUOTA) self.rest.init_cluster_memoryQuota( self.input.membase_settings.rest_username, self.input.membase_settings.rest_password, kv_quota - INDEX_QUOTA - FTS_QUOTA - 100) try: self.log.info("Initialize node with services {0}".format( self.start_node_services)) status = self.rest.init_node_services( hostname=self.master.ip, services=[self.start_node_services]) self.rest.init_cluster() except Exception as e: if e: print(e) if not status: if self.version not in COUCHBASE_FROM_WATSON and \ self.start_node_services not in sherlock_services_in_ce: self.log.info( "initial services setting enforced in Sherlock CE") elif self.version in COUCHBASE_FROM_WATSON and \ self.start_node_services not in watson_services_in_ce: self.log.info("initial services setting enforced in Watson CE") elif status: add_node = False try: self.log.info("node with services {0} try to add".format( self.add_node_services)) add_node = self.cluster.rebalance( self.servers[:2], self.servers[1:2], [], services=[self.add_node_services]) except Exception: pass if add_node: self.get_services_map() list_nodes = self.get_nodes_from_services_map( get_all_nodes=True) map = self.get_nodes_services() if map[self.master.ip] == self.start_node_services and \ map[self.servers[1].ip] == self.add_node_services: self.log.info( "services set correctly when node added & rebalance") else: self.fail("services set incorrectly when node added & rebalance. " "cluster expected services: {0}; set cluster services {1} ." "add node expected srv: {2}; set add node srv {3}"\ .format(map[self.master.ip], self.start_node_services, \ map[self.servers[1].ip], self.add_node_services)) else: if self.version not in COUCHBASE_FROM_WATSON: if self.start_node_services in ["kv", "index,kv,n1ql"] and \ self.add_node_services not in ["kv", "index,kv,n1ql"]: self.log.info("services are enforced in CE") elif self.start_node_services not in [ "kv", "index,kv,n1ql" ]: self.log.info("services are enforced in CE") else: self.fail("maybe bug in add node") elif self.version in COUCHBASE_FROM_WATSON: if self.start_node_services in ["kv", "index,kv,n1ql", "fts,index,kv,n1ql"] and self.add_node_services not in \ ["kv", "index,kv,n1ql", "fts,index,kv,n1ql"]: self.log.info("services are enforced in CE") elif self.start_node_services not in [ "kv", "index,kv,n1ql", "fts,index,kv,n1ql" ]: self.log.info("services are enforced in CE") else: self.fail("maybe bug in add node") else: self.fail("maybe bug in node initialization") def check_full_backup_only(self): """ for windows vm, ask IT to put uniq.exe at /cygdrive/c/Program Files (x86)/ICW/bin directory """ self.remote = RemoteMachineShellConnection(self.master) """ put params items=0 in test param so that init items = 0 """ self.remote.execute_command("{0}cbworkloadgen -n {1}:8091 -j -i 1000 " \ "-u Administrator -p password" \ .format(self.bin_path, self.master.ip)) """ delete backup location before run backup """ self.remote.execute_command("rm -rf {0}*".format(self.backup_location)) output, error = self.remote.execute_command("ls -lh {0}".format( self.backup_location)) self.remote.log_command_output(output, error) """ first full backup """ self.remote.execute_command("{0}cbbackup http://{1}:8091 {2} -m full " \ "-u Administrator -p password"\ .format(self.bin_path, self.master.ip, self.backup_c_location)) output, error = self.remote.execute_command("ls -lh {0}*/".format( self.backup_location)) self.remote.log_command_output(output, error) output, error = self.remote.execute_command("{0}cbtransfer -u Administrator "\ "-p password {1}*/*-full/ " \ "stdout: | grep set | uniq | wc -l"\ .format(self.bin_path, self.backup_c_location)) self.remote.log_command_output(output, error) if int(output[0]) != 1000: self.fail("full backup did not work in CE. " "Expected 1000, actual: {0}".format(output[0])) self.remote.execute_command("{0}cbworkloadgen -n {1}:8091 -j -i 1000 "\ " -u Administrator -p password --prefix=t_" .format(self.bin_path, self.master.ip)) """ do different backup mode """ self.remote.execute_command("{0}cbbackup -u Administrator -p password "\ "http://{1}:8091 {2} -m {3}"\ .format(self.bin_path, self.master.ip, self.backup_c_location, self.backup_option)) output, error = self.remote.execute_command("ls -lh {0}".format( self.backup_location)) self.remote.log_command_output(output, error) output, error = self.remote.execute_command("{0}cbtransfer -u Administrator "\ "-p password {1}*/*-{2}/ stdout: "\ "| grep set | uniq | wc -l"\ .format(self.bin_path, self.backup_c_location, self.backup_option)) self.remote.log_command_output(output, error) if int(output[0]) == 2000: self.log.info("backup option 'diff' is enforced in CE") elif int(output[0]) == 1000: self.fail("backup option 'diff' is not enforced in CE. " "Expected 2000, actual: {0}".format(output[0])) else: self.fail("backup failed to backup correct items") self.remote.disconnect() def check_ent_backup(self): """ for CE version from Watson, cbbackupmgr exe file should not in bin """ command = "cbbackupmgr" self.remote = RemoteMachineShellConnection(self.master) self.log.info("check if {0} in {1} directory".format( command, self.bin_path)) found = self.remote.file_exists(self.bin_path, command) if found: self.log.info("found {0} in {1} directory".format( command, self.bin_path)) self.log.info("Ent. backup in CE is in bin!") elif not found: self.fail( "CE from Cheshire Cat should contain {0}".format(command)) self.remote.disconnect() def check_memory_optimized_storage_mode(self): """ from Watson, CE should not have option 'memory_optimized' to set """ self.rest.force_eject_node() self.sleep(5, "wait for node reset done") try: self.log.info("Initialize node with 'Memory Optimized' option") status = self.rest.set_indexer_storage_mode( username=self.input.membase_settings.rest_username, password=self.input.membase_settings.rest_password, storageMode='memory_optimized') except Exception as ex: if ex: print(ex) if not status: self.log.info("Memory Optimized setting enforced in CE " "Could not set memory_optimized option") else: self.fail("Memory Optimzed setting does not enforced in CE " "We could set this option in") def check_plasma_storage_mode(self): """ from Watson, CE should not have option 'memory_optimized' to set """ self.rest.force_eject_node() self.sleep(5, "wait for node reset done") try: self.log.info("Initialize node with 'Memory Optimized' option") status = self.rest.set_indexer_storage_mode( username=self.input.membase_settings.rest_username, password=self.input.membase_settings.rest_password, storageMode='plasma') except Exception as ex: if ex: print(ex) if not status: self.log.info("Plasma setting enforced in CE " "Could not set Plasma option") else: self.fail("Plasma setting does not enforced in CE " "We could set this option in") def check_x509_cert(self): """ from Watson, X509 certificate only support in EE """ api = self.rest.baseUrl + "pools/default/certificate?extended=true" self.log.info("request to get certificate at " "'pools/default/certificate?extended=true' " "should return False") try: status, content, header = self.rest._http_request(api, 'GET') except Exception as ex: if ex: print(ex) if status: self.fail("This X509 certificate feature only available in EE") elif not status: if b'requires enterprise edition' in content: self.log.info("X509 cert is enforced in CE") def check_roles_base_access(self): """ from Watson, roles base access for admin should not in in CE """ if self.user_add is None: self.fail( "We need to pass user name (user_add) to run this test. ") if self.user_role is None: self.fail( "We need to pass user roles (user_role) to run this test. ") api = self.rest.baseUrl + "settings/rbac/users/" + self.user_add self.log.info("url to run this test: %s" % api) """ add admin user """ param = "name=%s&roles=%s" % (self.user_add, self.user_role) try: status, content, header = self.rest._http_request( api, 'PUT', param) except Exception as ex: if ex: print(ex) if status: self.fail("CE should not allow to add admin users") else: self.log.info("roles base is enforced in CE! ") def check_root_certificate(self): """ from watson, ce should not see root certificate manual test: curl -u Administrator:password -X GET http://localhost:8091/pools/default/certificate """ api = self.rest.baseUrl + "pools/default/certificate" try: status, content, header = self.rest._http_request(api, 'GET') except Exception as ex: if ex: print(ex) if status: self.fail("CE should not see root certificate!") elif b'requires enterprise edition' in content: self.log.info("root certificate is enforced in CE! ") def check_settings_audit(self): """ from watson, ce should not set audit manual test: curl -u Administrator:password -X GET http://localhost:8091/settings/audit """ api = self.rest.baseUrl + "settings/audit" try: status, content, header = self.rest._http_request(api, 'GET') except Exception as ex: if ex: print(ex) if status: self.fail("CE should not allow to set audit !") elif b'requires enterprise edition' in content: self.log.info("settings audit is enforced in CE! ") def check_infer(self): """ from watson, ce should not see infer manual test: curl -H "Content-Type: application/json" -X POST -d '{"statement":"infer `bucket_name`;"}' http://localhost:8093/query/service test params: new_services=kv-index-n1ql,default_bucket=False """ self.rest.force_eject_node() self.sleep(7, "wait for node reset done") self.rest.init_node() bucket = "default" self.rest.create_bucket(bucket, ramQuotaMB=200) api = self.rest.query_baseUrl + "query/service" param = urllib.parse.urlencode({"statement": "infer `%s` ;" % bucket}) try: status, content, header = self.rest._http_request( api, 'POST', param) json_parsed = json.loads(content) except Exception as ex: if ex: print(ex) if json_parsed["status"] == "success": self.fail("CE should not allow to run INFER !") elif json_parsed["status"] == "fatal": self.log.info("INFER is enforced in CE! ") def check_query_monitoring(self): self.rest.force_eject_node() self.sleep(7, "wait for node reset done") self.rest.init_node() bucket = "default" self.rest.create_bucket(bucket, ramQuotaMB=200) api = self.rest.query_baseUrl + "admin/settings" param = {'profile': 'phases'} try: status, content, header = self.rest._http_request( api, 'POST', json.dumps(param)) except Exception as ex: if ex: print(ex) if status: self.fail("CE should not be allowed to do query monitoring !") elif b'Profiling is an EE only feature' in content: self.log.info("Query monitoring is enforced in CE! ") def check_flex_index(self): """ from watson, ce should not see infer manual test: curl -H "Content-Type: application/json" -X POST -d '{"statement":"infer `bucket_name`;"}' http://localhost:8093/query/service test params: new_services=kv-index-n1ql,default_bucket=False """ self.rest.force_eject_node() self.sleep(7, "wait for node reset done") self.rest.init_node() bucket = "default" self.rest.create_bucket(bucket, ramQuotaMB=200) api = self.rest.query_baseUrl + "query/service" param = urllib.parse.urlencode({ "statement": "SELECT META(d).id FROM `%s` AS d USE INDEX (USING FTS) WHERE d.f2 = 100;" % bucket }) try: status, content, header = self.rest._http_request( api, 'POST', param) json_parsed = json.loads(content) except Exception as ex: if ex: print(ex) if json_parsed["status"] == "success": self.fail("CE should not allow to run flex index !") elif json_parsed["status"] == "fatal": self.log.info("Flex index is enforced in CE! ") def check_index_partitioning(self): self.rest.force_eject_node() self.sleep(7, "wait for node reset done") self.rest.init_node() bucket = "default" self.rest.create_bucket(bucket, ramQuotaMB=200) api = self.rest.query_baseUrl + "query/service" param = urllib.parse.urlencode({ "statement": "CREATE INDEX idx ON `%s`(id) PARTITION BY HASH(META().id)" % bucket }) try: status, content, header = self.rest._http_request( api, 'POST', param) json_parsed = json.loads(content) except Exception as ex: if ex: print(ex) if json_parsed["status"] == "success": self.fail("CE should not be allowed to run index partitioning !") elif json_parsed["status"] == "fatal": self.log.info("Index partitioning is enforced in CE! ") def check_query_cost_based_optimizer(self): self.rest.force_eject_node() self.sleep(7, "wait for node reset done") self.rest.init_node() bucket = "default" self.rest.create_bucket(bucket, ramQuotaMB=200) api = self.rest.query_baseUrl + "query/service" param = urllib.parse.urlencode({ "statement": "UPDATE STATISTICS for `hotel` (type, address, city, country, free_breakfast, id, phone);" }) try: status, content, header = self.rest._http_request( api, 'POST', param) json_parsed = json.loads(content) except Exception as ex: if ex: print(ex) if json_parsed["status"] == "success": self.fail("CE should not be allowed to run CBO !") elif json_parsed["status"] == "fatal": self.log.info("CBO is enforced in CE! ") def check_query_window_functions(self): self.rest.force_eject_node() self.sleep(7, "wait for node reset done") self.rest.init_node() bucket = "default" self.rest.create_bucket(bucket, ramQuotaMB=200) api = self.rest.query_baseUrl + "query/service" param = urllib.parse.urlencode({ "statement": "SELECT d.id, d.destinationairport, CUME_DIST() OVER (PARTITION BY d.destinationairport \ ORDER BY d.distance NULLS LAST) AS `rank` \ FROM `%s` AS d \ WHERE d.type='route' \ LIMIT 7;" % bucket }) try: status, content, header = self.rest._http_request( api, 'POST', param) json_parsed = json.loads(content) except Exception as ex: if ex: print(ex) if json_parsed["status"] == "success": self.fail("CE should not be allowed to use window functions !") elif json_parsed["status"] == "fatal": self.log.info("Window functions is enforced in CE! ") def check_auto_complete(self): """ this feature has not complete to block in CE """ """ Check new features from spock start here """ def check_cbbackupmgr(self): """ cbbackupmgr should not available in CE from spock """ if self.cb_version[:5] in COUCHBASE_FROM_SPOCK: file_name = "cbbackupmgr" + self.file_extension self.log.info("check if cbbackupmgr in bin dir in CE") result = self.remote.file_exists(self.bin_path, file_name) if result: self.fail("cbbackupmgr should not in bin dir of CE") else: self.log.info("cbbackupmgr is enforced in CE") self.remote.disconnect() def test_max_ttl_bucket(self): """ From vulcan, EE bucket has has an option to set --max-ttl, not it CE. This test is make sure CE could not create bucket with option --max-ttl This test must pass default_bucket=False """ if self.cb_version[:5] not in COUCHBASE_FROM_VULCAN: self.log.info("This test only for vulcan and later") return cmd = 'curl -X POST -u Administrator:password \ http://{0}:8091/pools/default/buckets \ -d name=bucket0 \ -d maxTTL=100 \ -d ramQuotaMB=100 '.format(self.master.ip) if self.cli_test: cmd = "{0}couchbase-cli bucket-create -c {1}:8091 --username Administrator \ --password password --bucket bucket0 --bucket-type couchbase \ --bucket-ramsize 512 --bucket-replica 1 --bucket-priority high \ --bucket-eviction-policy fullEviction --enable-flush 0 \ --enable-index-replica 1 --max-ttl 200".format( self.bin_path, self.master.ip) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "Max TTL is supported in enterprise edition only" if self.cli_test: mesg = "Maximum TTL can only be configured on enterprise edition" if output and mesg not in str(output[0]): self.fail("max ttl feature should not in Community Edition") buckets = RestConnection(self.master).get_buckets() if buckets: for bucket in buckets: self.log.info("bucekt in cluser: {0}".format(bucket.name)) if bucket.name == "bucket0": self.fail("Failed to enforce feature max ttl in CE.") conn.disconnect() def test_setting_audit(self): """ CE does not allow to set audit from vulcan 5.5.0 """ if self.cb_version[:5] not in COUCHBASE_FROM_VULCAN: self.log.info("This test only for vulcan and later") return cmd = 'curl -X POST -u Administrator:password \ http://{0}:8091/settings/audit \ -d auditdEnabled=true '.format(self.master.ip) if self.cli_test: cmd = "{0}couchbase-cli setting-audit -c {1}:8091 -u Administrator \ -p password --audit-enabled 1 --audit-log-rotate-interval 604800 \ --audit-log-path /opt/couchbase/var/lib/couchbase/logs --set"\ .format(self.bin_path, self.master.ip) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "This http API endpoint requires enterprise edition" if output and mesg not in str(output[0]): self.fail("setting-audit feature should not in Community Edition") conn.disconnect() def test_setting_autofailover_enterprise_only(self): """ CE does not allow set auto failover if disk has issue and failover group from vulcan 5.5.0 """ if self.cb_version[:5] not in COUCHBASE_FROM_VULCAN: self.log.info("This test only for vulcan and later") return self.failover_disk_period = self.input.param("failover_disk_period", False) self.failover_server_group = self.input.param("failover_server_group", False) failover_disk_period = "" if self.failover_disk_period: if self.cli_test: failover_disk_period = "--failover-data-disk-period 300" else: failover_disk_period = "-d failoverOnDataDiskIssues[timePeriod]=300" failover_server_group = "" if self.failover_server_group and self.cli_test: failover_server_group = "--enable-failover-of-server-group 1" cmd = 'curl -X POST -u Administrator:password \ http://{0}:8091/settings/autoFailover -d enabled=true -d timeout=120 \ -d maxCount=1 \ -d failoverOnDataDiskIssues[enabled]=true {1} \ -d failoverServerGroup={2}'.format(self.master.ip, failover_disk_period, self.failover_server_group) if self.cli_test: cmd = "{0}couchbase-cli setting-autofailover -c {1}:8091 \ -u Administrator -p password \ --enable-failover-on-data-disk-issues 1 {2} {3} "\ .format(self.bin_path, self.master.ip, failover_disk_period, failover_server_group) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "Auto failover on Data Service disk issues can only be " + \ "configured on enterprise edition" if not self.cli_test: if self.failover_disk_period or \ self.failover_server_group: if output and not error: self.fail("setting autofailover disk issues feature\ should not in Community Edition") else: if self.failover_server_group: mesg = "--enable-failover-of-server-groups can only be " + \ "configured on enterprise edition" if output and mesg not in str(output[0]): self.fail("Setting EE autofailover features \ should not in Community Edition") else: self.log.info("EE setting autofailover are disable in CE") conn.disconnect() def test_set_bucket_compression(self): """ CE does not allow to set bucket compression to bucket from vulcan 5.5.0. Mode compression: off,active,passive Note: must set defaultbucket=False for this test """ if self.cb_version[:5] not in COUCHBASE_FROM_VULCAN: self.log.info("This test only for vulcan and later") return self.compression_mode = self.input.param("compression_mode", "off") cmd = 'curl -X POST -u Administrator:password \ http://{0}:8091/pools/default/buckets \ -d name=bucket0 \ -d compressionMode={1} \ -d ramQuotaMB=100 '.format( self.master.ip, self.compression_mode) if self.cli_test: cmd = "{0}couchbase-cli bucket-create -c {1}:8091 --username Administrator \ --password password --bucket bucket0 --bucket-type couchbase \ --bucket-ramsize 512 --bucket-replica 1 --bucket-priority high \ --bucket-eviction-policy fullEviction --enable-flush 0 \ --enable-index-replica 1 --compression-mode {2}".format( self.bin_path, self.master.ip, self.compression_mode) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "Compression mode is supported in enterprise edition only" if self.cli_test: mesg = "Compression mode can only be configured on enterprise edition" if output and mesg not in str(output[0]): self.fail("Setting bucket compression should not in CE") conn.disconnect() def test_ldap_groups(self): """ LDAP Groups feature is not available in CE """ if self.cb_version[:5] not in COUCHBASE_FROM_MAD_HATTER: self.log.info("This test is only for MH and later") return cmd = 'curl -X POST -u Administrator:password \ http://{0}:8091/settings/rbac/groups/admins \ -d roles=admin \ -d description="Couchbase+Server+Administrators" \ --data-urlencode ldap_group_ref="uid=cbadmins,ou=groups,dc=example,dc=com"'\ .format(self.master.ip) if self.cli_test: cmd = '{0}couchbase-cli user-manage -c {1}:8091 --username Administrator \ --password password \ --set-group \ --group-name admins \ --roles admin \ --group-description "Couchbase Server Administrators" \ --ldap-ref "uid=cbadmins,ou=groups,dc=example,dc=com"'.format( self.bin_path, self.master.ip) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "Requested resource not found." if self.cli_test: mesg = "ERROR: This http API endpoint requires enterprise edition" if output and mesg not in str(output[0]): self.fail("LDAP Groups should not be in CE") conn.disconnect() def test_ldap_cert(self): """ LDAP Cert feature is not available in CE """ if self.cb_version[:5] not in COUCHBASE_FROM_MAD_HATTER: self.log.info("This test is only for MH and later") return cmd = 'curl -X POST -u Administrator:password http://{0}:8091/settings/ldap \ -d hosts={1} \ -d port=389 \ -d encryption=StartTLSExtension \ -d serverCertValidation=true \ --data-urlencode [email protected] \ -d bindDN="cn=admin,dc=example,dc=com" \ -d bindPass=password \ -d authenticationEnabled=true \ -d authorizationEnabled=true \ --data-urlencode groupsQuery="ou=groups,dc=example,dc=com??one?(member=%D)"'\ .format(self.master.ip, self.master.ip) if self.cli_test: cmd = '{0}couchbase-cli setting-ldap -c {1}:8091 --username Administrator \ --password password \ --authentication-enabled 1 \ --authorization-enabled 1 \ --hosts {2} \ --encryption startTLS \ --client-cert root.crt \ --bind-dn "cn=admin,dc=example,dc=com" \ --bind-password password \ --group-query "ou=groups,dc=example,dc=com??one?(member=%D)"'.format( self.bin_path, self.master.ip, self.master.ip) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "This http API endpoint requires enterprise edition" if self.cli_test: mesg = "ERROR: Command only available in enterprise edition" if output and mesg not in str(output[0]): self.fail("LDAP Cert should not be in CE") conn.disconnect() def test_network_encryption(self): """ Encrypted network access is not available in CE """ if self.cb_version[:5] not in COUCHBASE_FROM_MAD_HATTER: self.log.info("This test is only for MH and later") return cmd = 'curl -u Administrator:password -v -X POST \ http://{0}:8091/settings/security \ -d disableUIOverHttp=true \ -d clusterEncryptionLevel=control \ -d tlsMinVersion=tlsv1.1 \ -d "cipherSuites=["TLS_RSA_WITH_AES_128_CBC_SHA", "TLS_RSA_WITH_AES_256_CBC_SHA"]"'\ .format(self.master.ip) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "not supported in community edition" if output and mesg not in str(output[0]): self.fail("Encrypted network access should not be in CE") conn.disconnect() def test_n2n_encryption(self): """ Encrypted network access is not available in CE """ if self.cb_version[:5] not in COUCHBASE_FROM_MAD_HATTER: self.log.info("This test is only for MH and later") return cmd = '/opt/couchbase/bin/couchbase-cli node-to-node-encryption \ -c http://{0}:8091 \ -u Administrator \ -p password \ --enable'\ .format(self.master.ip) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "not supported in community edition" if output and mesg not in str(output[0]): self.fail("Encrypted network access should not be in CE") conn.disconnect() def test_log_redaction(self): """ Log redaction feature is not available in CE """ if self.cb_version[:5] not in COUCHBASE_FROM_MAD_HATTER: self.log.info("This test is only for MH and later") return cmd = 'curl -X POST -u Administrator:password \ http://{0}:8091/controller/startLogsCollection \ -d nodes="*" \ -d logRedactionLevel=partial'.format( self.master.ip) if self.cli_test: cmd = '{0}couchbase-cli collect-logs-start -c {1}:8091 --username Administrator \ --password password \ --all-nodes \ --redaction-level partial'.format( self.bin_path, self.master.ip) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "log redaction is an enterprise only feature" if output and mesg not in str(output[0]): self.fail("Log redaction should not be in CE") conn.disconnect()
class AutoRetryFailedRebalance(RebalanceBaseTest): def setUp(self): super(AutoRetryFailedRebalance, self).setUp() self.rest = RestConnection(self.cluster.master) self.sleep_time = self.input.param("sleep_time", 15) self.enabled = self.input.param("enabled", True) self.afterTimePeriod = self.input.param("afterTimePeriod", 300) self.maxAttempts = self.input.param("maxAttempts", 1) self.log.info("Changing the retry rebalance settings ....") self.change_retry_rebalance_settings( enabled=self.enabled, afterTimePeriod=self.afterTimePeriod, maxAttempts=self.maxAttempts) self.rebalance_operation = self.input.param("rebalance_operation", "rebalance_out") self.disable_auto_failover = self.input.param("disable_auto_failover", True) self.auto_failover_timeout = self.input.param("auto_failover_timeout", 120) if self.disable_auto_failover: self.rest.update_autofailover_settings(False, 120) else: self.rest.update_autofailover_settings(True, self.auto_failover_timeout) self.cb_collect_failure_nodes = dict() # To support data load during auto retry op self.data_load = self.input.param("data_load", False) self.rebalance_failed_msg = "Rebalance failed as expected" def tearDown(self): self.reset_retry_rebalance_settings() self.cbcollect_info() # Reset to default value super(AutoRetryFailedRebalance, self).tearDown() rest = RestConnection(self.servers[0]) zones = rest.get_zone_names() for zone in zones: if zone != "Group 1": rest.delete_zone(zone) def __update_cbcollect_expected_node_failures(self, nodes, reason): for node in nodes: self.cb_collect_failure_nodes[node.ip] = reason def set_retry_exceptions(self, doc_loading_spec): retry_exceptions = list() retry_exceptions.append(SDKException.AmbiguousTimeoutException) retry_exceptions.append(SDKException.TimeoutException) retry_exceptions.append(SDKException.RequestCanceledException) retry_exceptions.append(SDKException.DocumentNotFoundException) retry_exceptions.append(SDKException.ServerOutOfMemoryException) if self.durability_level: retry_exceptions.append(SDKException.DurabilityAmbiguousException) retry_exceptions.append(SDKException.DurabilityImpossibleException) doc_loading_spec[MetaCrudParams.RETRY_EXCEPTIONS] = retry_exceptions def async_data_load(self): doc_loading_spec = self.bucket_util.get_crud_template_from_package( "volume_test_load") self.set_retry_exceptions(doc_loading_spec) tasks = self.bucket_util.run_scenario_from_spec( self.task, self.cluster, self.cluster.buckets, doc_loading_spec, mutation_num=0, async_load=True, batch_size=self.batch_size, process_concurrency=self.process_concurrency) return tasks def data_validation(self, tasks): self.task.jython_task_manager.get_task_result(tasks) self.bucket_util.validate_doc_loading_results(tasks) if tasks.result is False: self.fail("Doc_loading failed") self.cluster_util.print_cluster_stats(self.cluster) self.bucket_util._wait_for_stats_all_buckets(self.cluster, self.cluster.buckets, timeout=1200) self.bucket_util.validate_docs_per_collections_all_buckets( self.cluster) self.bucket_util.print_bucket_stats(self.cluster) def test_auto_retry_of_failed_rebalance_where_failure_happens_before_rebalance( self): tasks = None before_rebalance_failure = self.input.param("before_rebalance_failure", "stop_server") # induce the failure before the rebalance starts self._induce_error(before_rebalance_failure) self.sleep(self.sleep_time) try: rebalance = self._rebalance_operation(self.rebalance_operation) self.task.jython_task_manager.get_task_result(rebalance) self.assertTrue(rebalance.result, self.rebalance_failed_msg) except Exception as e: self.log.info("Rebalance failed with: {0}".format(str(e))) # Trigger cbcollect after rebalance failure self.cbcollect_info(trigger=True, validate=False, known_failures=self.cb_collect_failure_nodes) # Recover from the error self._recover_from_error(before_rebalance_failure) if self.data_load: tasks = self.async_data_load() self.check_retry_rebalance_succeeded() # Validate cbcollect result after rebalance retry self.cbcollect_info(trigger=False, validate=True, known_failures=self.cb_collect_failure_nodes) if self.data_load: self.data_validation(tasks) else: self.fail("Rebalance did not fail as expected. " "Hence could not validate auto-retry feature..") finally: if self.disable_auto_failover: self.rest.update_autofailover_settings(True, 120) self.cluster_util.start_server(self.cluster, self.servers[1]) self.cluster_util.stop_firewall_on_node(self.cluster, self.servers[1]) def test_auto_retry_of_failed_rebalance_where_failure_happens_during_rebalance( self): tasks = None during_rebalance_failure = self.input.param("during_rebalance_failure", "stop_server") try: rebalance = self._rebalance_operation(self.rebalance_operation) self.sleep(self.sleep_time) # induce the failure during the rebalance self._induce_error(during_rebalance_failure) self.task.jython_task_manager.get_task_result(rebalance) self.assertTrue(rebalance.result, self.rebalance_failed_msg) except Exception as e: self.log.info("Rebalance failed with: {0}".format(str(e))) # Recover from the error self._recover_from_error(during_rebalance_failure) if self.data_load: tasks = self.async_data_load() self.check_retry_rebalance_succeeded() if self.data_load: self.data_validation(tasks) else: # This is added as the failover task is not throwing exception if self.rebalance_operation == "graceful_failover": # Recover from the error self._recover_from_error(during_rebalance_failure) if self.data_load: tasks = self.async_data_load() self.check_retry_rebalance_succeeded() if self.data_load: self.data_validation(tasks) else: self.fail("Rebalance did not fail as expected. " "Hence could not validate auto-retry feature..") finally: if self.disable_auto_failover: self.rest.update_autofailover_settings(True, 120) self.cluster_util.start_server(self.cluster, self.servers[1]) self.cluster_util.stop_firewall_on_node(self.cluster, self.servers[1]) def test_auto_retry_of_failed_rebalance_does_not_get_triggered_when_rebalance_is_stopped( self): _ = self._rebalance_operation(self.rebalance_operation) reached = self.cluster_util.rebalance_reached(self.rest, 30) self.assertTrue(reached, "Rebalance failed or did not reach 30%") # Trigger cbcollect before interrupting the rebalance self.cbcollect_info(trigger=True, validate=False) self.rest.stop_rebalance(wait_timeout=self.sleep_time) result = json.loads(self.rest.get_pending_rebalance_info()) self.log.info(result) # Validate cbcollect results self.cbcollect_info(trigger=False, validate=True) retry_rebalance = result["retry_rebalance"] if retry_rebalance != "not_pending": self.fail( "Auto-retry succeeded even when Rebalance was stopped by user") def test_negative_auto_retry_of_failed_rebalance_where_rebalance_will_be_cancelled( self): during_rebalance_failure = self.input.param("during_rebalance_failure", "stop_server") post_failure_operation = self.input.param("post_failure_operation", "cancel_pending_rebalance") try: rebalance = self._rebalance_operation(self.rebalance_operation) self.sleep(self.sleep_time) # induce the failure during the rebalance self._induce_error(during_rebalance_failure) self.task.jython_task_manager.get_task_result(rebalance) self.assertTrue(rebalance.result, self.rebalance_failed_msg) except Exception as e: self.log.info("Rebalance failed with: %s" % e) # Recover from the error self._recover_from_error(during_rebalance_failure) # TODO : Data load at this stage fails; # if self.data_load: # tasks = self.async_data_load() result = json.loads(self.rest.get_pending_rebalance_info()) # if self.data_load: # self.data_validation(tasks) self.log.info(result) retry_rebalance = result["retry_rebalance"] rebalance_id = result["rebalance_id"] if retry_rebalance != "pending": self.fail("Auto-retry of failed rebalance is not triggered") if post_failure_operation == "cancel_pending_rebalance": # cancel pending rebalance self.log.info("Cancelling rebalance-id: %s" % rebalance_id) self.rest.cancel_pending_rebalance(rebalance_id) elif post_failure_operation == "disable_auto_retry": # disable the auto retry of the failed rebalance self.log.info("Disable the auto retry of the failed rebalance") self.change_retry_rebalance_settings(enabled=False) elif post_failure_operation == "retry_failed_rebalance_manually": # retry failed rebalance manually self.log.info("Retrying failed rebalance id %s" % rebalance_id) self.cluster.rebalance(self.servers[:self.nodes_init], [], []) else: self.fail("Invalid post_failure_operation option") # Now check and ensure retry won't happen result = json.loads(self.rest.get_pending_rebalance_info()) self.log.info(result) retry_rebalance = result["retry_rebalance"] if retry_rebalance != "not_pending": self.fail("Auto-retry of failed rebalance is not cancelled") else: self.fail("Rebalance did not fail as expected. " "Hence could not validate auto-retry feature..") finally: if self.disable_auto_failover: self.rest.update_autofailover_settings(True, 120) self.cluster_util.start_server(self.cluster, self.servers[1]) self.cluster_util.stop_firewall_on_node(self.cluster, self.servers[1]) def test_negative_auto_retry_of_failed_rebalance_where_rebalance_will_not_be_cancelled( self): during_rebalance_failure = self.input.param("during_rebalance_failure", "stop_server") post_failure_operation = self.input.param("post_failure_operation", "create_delete_buckets") zone_name = "Group_{0}_{1}".format(random.randint(1, 1000000000), self._testMethodName) zone_name = zone_name[0:60] default_zone = "Group 1" moved_node = [self.servers[1].ip] try: rebalance = self._rebalance_operation(self.rebalance_operation) self.sleep(self.sleep_time) # induce the failure during the rebalance self._induce_error(during_rebalance_failure) self.task.jython_task_manager.get_task_result(rebalance) self.assertTrue(rebalance.result, self.rebalance_failed_msg) except Exception as e: self.log.info("Rebalance failed with : {0}".format(str(e))) # Recover from the error self._recover_from_error(during_rebalance_failure) result = json.loads(self.rest.get_pending_rebalance_info()) self.log.info(result) retry_rebalance = result["retry_rebalance"] if retry_rebalance != "pending": self.fail("Auto-retry of failed rebalance is not triggered") # if post_failure_operation == "create_delete_buckets": # # delete buckets and create new one # BucketOperationHelper.delete_all_buckets_or_assert(servers=self.servers, test_case=self) # self.sleep(self.sleep_time) # BucketOperationHelper.create_bucket(self.master, test_case=self) # Start cbcollect only if auto-retry of rebalance is triggered self.cbcollect_info(trigger=True, validate=False) if post_failure_operation == "change_replica_count": # change replica count self.log.info("Changing replica count of buckets") for bucket in self.cluster.buckets: self.bucket_util.update_bucket_property( self.cluster.master, bucket, replica_number=2) elif post_failure_operation == "change_server_group": # change server group self.log.info("Creating new zone " + zone_name) self.rest.add_zone(zone_name) self.log.info("Moving {0} to new zone {1}".format( moved_node, zone_name)) _ = self.rest.shuffle_nodes_in_zones(moved_node, default_zone, zone_name) else: self.fail("Invalid post_failure_operation option") # In these failure scenarios while the retry is pending, # then the retry will be attempted but fail try: self.check_retry_rebalance_succeeded() # Validate cbcollect results self.cbcollect_info(trigger=False, validate=True) except Exception as e: self.log.info(e) # Wait for cbstat to complete before asserting self.cbcollect_info(trigger=False, validate=True) if "Retrying of rebalance still did not help. All the retries exhausted" not in str( e): self.fail( "Auto retry of failed rebalance succeeded when it was expected to fail" ) else: self.fail( "Rebalance did not fail as expected. Hence could not validate auto-retry feature.." ) finally: if post_failure_operation == "change_server_group": status = self.rest.shuffle_nodes_in_zones( moved_node, zone_name, default_zone) self.log.info("Shuffle the node back to default group. " "Status: %s" % status) self.sleep(self.sleep_time) self.log.info("Deleting new zone " + zone_name) try: self.rest.delete_zone(zone_name) except: self.log.info("Errors in deleting zone") if self.disable_auto_failover: self.rest.update_autofailover_settings(True, 120) self.cluster_util.start_server(self.cluster, self.servers[1]) self.cluster_util.stop_firewall_on_node(self.cluster, self.servers[1]) def test_auto_retry_of_failed_rebalance_with_rebalance_test_conditions( self): tasks = None test_failure_condition = self.input.param("test_failure_condition") # induce the failure before the rebalance starts self._induce_rebalance_test_condition(test_failure_condition) self.sleep(self.sleep_time) try: rebalance = self._rebalance_operation(self.rebalance_operation) self.task.jython_task_manager.get_task_result(rebalance) self.assertTrue(rebalance.result, self.rebalance_failed_msg) except Exception as e: self.log.info("Rebalance failed with: %s" % e) # Delete the rebalance test condition to recover from the error self._delete_rebalance_test_condition(test_failure_condition) if self.data_load: tasks = self.async_data_load() self.check_retry_rebalance_succeeded() if self.data_load: self.data_validation(tasks) else: self.fail("Rebalance did not fail as expected. " "Hence could not validate auto-retry feature..") finally: if self.disable_auto_failover: self.rest.update_autofailover_settings(True, 120) self._delete_rebalance_test_condition(test_failure_condition) def test_auto_retry_of_failed_rebalance_with_autofailvoer_enabled(self): before_rebalance_failure = self.input.param("before_rebalance_failure", "stop_server") # induce the failure before the rebalance starts self._induce_error(before_rebalance_failure) try: rebalance = self._rebalance_operation(self.rebalance_operation) self.task.jython_task_manager.get_task_result(rebalance) self.assertTrue(rebalance.result, self.rebalance_failed_msg) except Exception as e: self.log.info("Rebalance failed with: {0}".format(str(e))) self.cbcollect_info(trigger=True, validate=False, known_failures=self.cb_collect_failure_nodes) if self.auto_failover_timeout < self.afterTimePeriod: self.sleep(self.auto_failover_timeout) result = json.loads(self.rest.get_pending_rebalance_info()) self.log.info(result) retry_rebalance = result["retry_rebalance"] if retry_rebalance != "not_pending": # Wait for cbcollect to complete before asserting self.cbcollect_info( trigger=False, validate=True, known_failures=self.cb_collect_failure_nodes) self.fail("Auto-failover did not cancel pending retry " "of the failed rebalance") else: try: self.check_retry_rebalance_succeeded() except Exception as e: expected_msg = "Retrying of rebalance still did not help" if expected_msg not in str(e): self.fail("Retry rebalance succeeded " "even without failover") self.sleep(self.auto_failover_timeout) self.cluster.rebalance(self.servers[:self.nodes_init], [], []) finally: self.cbcollect_info( trigger=False, validate=True, known_failures=self.cb_collect_failure_nodes) else: self.fail("Rebalance did not fail as expected. " "Hence could not validate auto-retry feature..") finally: if self.disable_auto_failover: self.rest.update_autofailover_settings(True, 120) self.cluster_util.start_server(self.cluster, self.servers[1]) self.cluster_util.stop_firewall_on_node(self.cluster, self.servers[1]) def test_cbcollect_with_rebalance_delay_condition(self): test_failure_condition = self.input.param("test_failure_condition") vb_num = self.input.param("target_vb") delay_milliseconds = self.input.param("delay_time", 60) * 1000 # induce the failure before the rebalance starts self._induce_rebalance_test_condition(test_failure_condition, vb_num=vb_num, delay_time=delay_milliseconds) self.sleep(self.sleep_time, "Wait for rebalance_test_condition to take effect") rebalance = self._rebalance_operation(self.rebalance_operation) # Start and validate cbcollect with rebalance delay self.cbcollect_info(trigger=True, validate=True) self.task.jython_task_manager.get_task_result(rebalance) if self.disable_auto_failover: self.rest.update_autofailover_settings(True, 120) self._delete_rebalance_test_condition(test_failure_condition) if rebalance.result is False: self.fail("Rebalance failed with test_condition: %s" % test_failure_condition) def _rebalance_operation(self, rebalance_operation): operation = None self.log.info("Starting rebalance operation of type: %s" % rebalance_operation) if rebalance_operation == "rebalance_out": operation = self.task.async_rebalance( self.cluster, [], self.cluster.servers[1:], retry_get_process_num=self.retry_get_process_num) self.__update_cbcollect_expected_node_failures( self.cluster.servers[1:], "out_node") elif rebalance_operation == "rebalance_in": operation = self.task.async_rebalance( self.cluster, [self.cluster.servers[self.nodes_init]], [], retry_get_process_num=self.retry_get_process_num) self.__update_cbcollect_expected_node_failures( [self.cluster.servers[self.nodes_init]], "in_node") elif rebalance_operation == "swap_rebalance": self.rest.add_node(self.cluster.master.rest_username, self.cluster.master.rest_password, self.cluster.servers[self.nodes_init].ip, self.cluster.servers[self.nodes_init].port) operation = self.task.async_rebalance( self.cluster, [], [self.cluster.servers[self.nodes_init - 1]], retry_get_process_num=self.retry_get_process_num) self.__update_cbcollect_expected_node_failures( [self.cluster.servers[self.nodes_init]], "in_node") self.__update_cbcollect_expected_node_failures( [self.cluster.servers[self.nodes_init - 1]], "out_node") elif rebalance_operation == "graceful_failover": # TODO : retry for graceful failover is not yet implemented operation = self.task.async_failover( [self.cluster.master], failover_nodes=[self.cluster.servers[1]], graceful=True, wait_for_pending=300) return operation def _induce_error(self, error_condition): cb_collect_err_str = None if error_condition == "stop_server": cb_collect_err_str = "failed" self.cluster_util.stop_server(self.cluster, self.servers[1]) elif error_condition == "enable_firewall": cb_collect_err_str = "failed" self.cluster_util.start_firewall_on_node(self.cluster, self.servers[1]) elif error_condition == "kill_memcached": self.cluster_util.kill_memcached(self.cluster, node=self.servers[1]) elif error_condition == "reboot_server": cb_collect_err_str = "failed" shell = RemoteMachineShellConnection(self.servers[1]) shell.reboot_node() shell.disconnect() elif error_condition == "kill_erlang": cb_collect_err_str = "failed" shell = RemoteMachineShellConnection(self.servers[1]) shell.kill_erlang() shell.disconnect() self.sleep(self.sleep_time * 3) else: self.fail("Invalid error induce option") if cb_collect_err_str: self.__update_cbcollect_expected_node_failures([self.servers[1]], cb_collect_err_str) def _recover_from_error(self, error_condition): if error_condition == "stop_server" \ or error_condition == "kill_erlang": self.cluster_util.start_server(self.cluster, self.servers[1]) elif error_condition == "enable_firewall": self.cluster_util.stop_firewall_on_node(self.cluster, self.servers[1]) elif error_condition == "reboot_server": self.sleep(self.sleep_time * 4) self.cluster_util.stop_firewall_on_node(self.cluster, self.servers[1]) def _induce_rebalance_test_condition(self, test_failure_condition, bucket_name="default", vb_num=1, delay_time=60000): if test_failure_condition == "verify_replication": set_command = 'testconditions:set(verify_replication, ' \ '{fail, "%s"})' % bucket_name elif test_failure_condition == "backfill_done": set_command = 'testconditions:set(backfill_done, ' \ '{for_vb_move, "%s", %s , fail})' \ % (bucket_name, vb_num) elif test_failure_condition == "delay_rebalance_start": set_command = 'testconditions:set(rebalance_start, {delay, %s}).' \ % delay_time elif test_failure_condition == "delay_verify_replication": set_command = 'testconditions:set(verify_replication, ' \ '{delay, "%s", %s})' % (bucket_name, delay_time) elif test_failure_condition == "delay_backfill_done": set_command = 'testconditions:set(backfill_done, ' \ '{for_vb_move, "%s", %s, {delay, %s}})' \ % (bucket_name, vb_num, delay_time) else: set_command = "testconditions:set(%s, fail)" \ % test_failure_condition get_command = "testconditions:get(%s)" % test_failure_condition for server in self.servers: rest = RestConnection(server) shell = RemoteMachineShellConnection(server) shell.enable_diag_eval_on_non_local_hosts() _, content = rest.diag_eval(set_command) self.log.debug("Set Command: %s. Return: %s" % (set_command, content)) shell.disconnect() for server in self.servers: rest = RestConnection(server) shell = RemoteMachineShellConnection(server) shell.enable_diag_eval_on_non_local_hosts() _, content = rest.diag_eval(get_command) self.log.info("Command: %s, Return: %s" % (get_command, content)) def _delete_rebalance_test_condition(self, test_failure_condition): if test_failure_condition.startswith("delay_"): test_failure_condition = test_failure_condition[6:] delete_command = "testconditions:delete(%s)" % test_failure_condition get_command = "testconditions:get(%s)" % test_failure_condition for server in self.servers: rest = RestConnection(server) shell = RemoteMachineShellConnection(server) shell.enable_diag_eval_on_non_local_hosts() _, content = rest.diag_eval(delete_command) self.log.info("Command: %s, Return: %s" % (delete_command, content)) for server in self.servers: rest = RestConnection(server) shell = RemoteMachineShellConnection(server) shell.enable_diag_eval_on_non_local_hosts() _, content = rest.diag_eval(get_command) self.log.info("Command: %s, Return: %s" % (get_command, content))
try: self.log.info("try to create zone {0} when cluster is not completely EE".format(zone_name)) result = rest.add_zone(zone_name) if result: raise Exception("Zone feature should not be available in CE version") except Exception,e : if "Failed" in e: pass serverInfo = self.servers[1] rest = RestConnection(serverInfo) self.user = serverInfo.rest_username self.password = serverInfo.rest_password if not rest.is_enterprise_edition(): raise Exception("Test failed to upgrade cluster from CE to EE") self.log.info("try to create zone {0} when cluster {1} is completely EE".format(zone_name, serverInfo.ip)) result = rest.add_zone(zone_name) self.log.info("sleep 5 seconds") time.sleep(5) if result: self.log.info("Zone feature is available in this cluster") else: raise Exception("Could not create zone with name: %s in cluster. It's a bug" % zone_name) if rest.is_zone_exist(zone_name.strip()): self.log.info("verified! zone '{0}' is existed".format(zone_name.strip())) else: raise Exception("There is not zone with name: %s in cluster. It's a bug" % zone_name) """ re-install enterprise edition for next test if there is any """ InstallerJob().parallel_install([self.servers[0]], params) """ reset master node to new node to teardown cluster """
def test_settingsCluster(self): ops = self.input.param("ops", None) source = 'ns_server' user = self.master.rest_username password = self.master.rest_password rest = RestConnection(self.master) if (ops == 'memoryQuota'): expectedResults = {'memory_quota':512, 'source':source, 'user':user, 'ip':self.ipAddress, 'port':12345, 'cluster_name':'', 'index_memory_quota':512,'fts_memory_quota': 302} rest.init_cluster_memoryQuota(expectedResults['user'], password, expectedResults['memory_quota']) elif (ops == 'loadSample'): expectedResults = {'name':'gamesim-sample', 'source':source, "user":user, 'ip':self.ipAddress, 'port':12345} rest.addSamples() #Get a REST Command for loading sample elif (ops == 'enableAutoFailover'): expectedResults = {'max_nodes':1, "timeout":120, 'source':source, "user":user, 'ip':self.ipAddress, 'port':12345} rest.update_autofailover_settings(True, expectedResults['timeout']) elif (ops == 'disableAutoFailover'): expectedResults = {'max_nodes':1, "timeout":120, 'source':source, "user":user, 'ip':self.ipAddress, 'port':12345} rest.update_autofailover_settings(False, expectedResults['timeout']) elif (ops == 'resetAutoFailover'): expectedResults = {'source':source, "user":user, 'ip':self.ipAddress, 'port':12345} rest.reset_autofailover() elif (ops == 'enableClusterAlerts'): expectedResults = {"encrypt":False, "email_server:port":25, "host":"localhost", "email_server:user":"******", "alerts":["auto_failover_node", "auto_failover_maximum_reached"], \ "recipients":["*****@*****.**"], "sender":"*****@*****.**", "source":"ns_server", "user":"******", 'ip':self.ipAddress, 'port':1234} rest.set_alerts_settings('*****@*****.**', '*****@*****.**', 'ritam', 'password',) elif (ops == 'disableClusterAlerts'): rest.set_alerts_settings('*****@*****.**', '*****@*****.**', 'ritam', 'password',) expectedResults = {'source':source, "user":user, 'ip':self.ipAddress, 'port':1234} rest.disable_alerts() elif (ops == 'modifyCompactionSettingsPercentage'): expectedResults = {"parallel_db_and_view_compaction":False, "database_fragmentation_threshold:percentage":50, "view_fragmentation_threshold:percentage":50, "purge_interval":3, "source":"ns_server", "user":"******", 'source':source, "user":user, 'ip':self.ipAddress, 'port':1234} rest.set_auto_compaction(dbFragmentThresholdPercentage=50, viewFragmntThresholdPercentage=50) elif (ops == 'modifyCompactionSettingsPercentSize'): expectedResults = {"parallel_db_and_view_compaction":False, "database_fragmentation_threshold:percentage":50, "database_fragmentation_threshold:size":10, "view_fragmentation_threshold:percentage":50, "view_fragmentation_threshold:size":10, "purge_interval":3, "source":"ns_server", "user":"******", 'source':source, "user":user, 'ip':self.ipAddress, 'port':1234} rest.set_auto_compaction(dbFragmentThresholdPercentage=50, viewFragmntThresholdPercentage=50, dbFragmentThreshold=10, viewFragmntThreshold=10) elif (ops == 'modifyCompactionSettingsTime'): expectedResults = {"parallel_db_and_view_compaction":False, "database_fragmentation_threshold:percentage":50, "database_fragmentation_threshold:size":10, "view_fragmentation_threshold:percentage":50, "view_fragmentation_threshold:size":10, "allowed_time_period:abort_outside":True, "allowed_time_period:to_minute":15, "allowed_time_period:from_minute":12, "allowed_time_period:to_hour":1, "allowed_time_period:from_hour":1, "purge_interval":3, "source":"ns_server", "user":"******", 'source':source, "user":user, 'ip':self.ipAddress, 'port':1234, } rest.set_auto_compaction(dbFragmentThresholdPercentage=50, viewFragmntThresholdPercentage=50, dbFragmentThreshold=10, viewFragmntThreshold=10, allowedTimePeriodFromHour=1, allowedTimePeriodFromMin=12, allowedTimePeriodToHour=1, allowedTimePeriodToMin=15, allowedTimePeriodAbort='true') elif (ops == "AddGroup"): expectedResults = {'group_name':'add group', 'source':source, 'user':user, 'ip':self.ipAddress, 'port':1234} rest.add_zone(expectedResults['group_name']) tempStr = rest.get_zone_uri()[expectedResults['group_name']] tempStr = (tempStr.split("/"))[4] expectedResults['uuid'] = tempStr elif (ops == "UpdateGroup"): expectedResults = {'group_name':'upGroup', 'source':source, 'user':user, 'ip':self.ipAddress, 'port':1234, 'nodes':[]} rest.add_zone(expectedResults['group_name']) rest.rename_zone(expectedResults['group_name'], 'update group') expectedResults['group_name'] = 'update group' tempStr = rest.get_zone_uri()[expectedResults['group_name']] tempStr = (tempStr.split("/"))[4] expectedResults['uuid'] = tempStr elif (ops == "UpdateGroupAddNodes"): sourceGroup = "Group 1" destGroup = 'destGroup' expectedResults = {'group_name':destGroup, 'source':source, 'user':user, 'ip':self.ipAddress, 'port':1234, 'nodes':['ns_1@' + self.master.ip], 'port':1234} #rest.add_zone(sourceGroup) rest.add_zone(destGroup) self.sleep(30) rest.shuffle_nodes_in_zones([self.master.ip], sourceGroup, destGroup) tempStr = rest.get_zone_uri()[expectedResults['group_name']] tempStr = (tempStr.split("/"))[4] expectedResults['uuid'] = tempStr elif (ops == "DeleteGroup"): expectedResults = {'group_name':'delete group', 'source':source, 'user':user, 'ip':self.ipAddress, 'port':1234} rest.add_zone(expectedResults['group_name']) tempStr = rest.get_zone_uri()[expectedResults['group_name']] tempStr = (tempStr.split("/"))[4] expectedResults['uuid'] = tempStr rest.delete_zone(expectedResults['group_name']) elif (ops == "regenCer"): expectedResults = {'source':source, 'user':user, 'ip':self.ipAddress, 'port':1234} rest.regenerate_cluster_certificate() elif (ops == 'renameNode'): rest.rename_node(self.master.ip, user, password) expectedResults = {"hostname":self.master.ip, "node":"ns_1@" + self.master.ip, "source":source, "user":user, "ip":self.ipAddress, "port":56845} try: self.checkConfig(self.eventID, self.master, expectedResults) finally: if (ops == "UpdateGroupAddNodes"): sourceGroup = "Group 1" destGroup = 'destGroup' rest.shuffle_nodes_in_zones([self.master.ip], destGroup, sourceGroup) rest = RestConnection(self.master) zones = rest.get_zone_names() for zone in zones: if zone != "Group 1": rest.delete_zone(zone)
def test_settingsCluster(self): ops = self.input.param("ops", None) source = 'ns_server' user = self.master.rest_username password = self.master.rest_password rest = RestConnection(self.master) if (ops == 'memoryQuota'): expectedResults = {'memory_quota':512, 'source':source, 'user':user, 'ip':self.ipAddress, 'port':12345, 'cluster_name':'', 'index_memory_quota':256} rest.init_cluster_memoryQuota(expectedResults['user'], password, expectedResults['memory_quota']) elif (ops == 'loadSample'): expectedResults = {'name':'gamesim-sample', 'source':source, "user":user, 'ip':self.ipAddress, 'port':12345} rest.addSamples() #Get a REST Command for loading sample elif (ops == 'enableAutoFailover'): expectedResults = {'max_nodes':1, "timeout":120, 'source':source, "user":user, 'ip':self.ipAddress, 'port':12345} rest.update_autofailover_settings(True, expectedResults['timeout']) elif (ops == 'disableAutoFailover'): expectedResults = {'max_nodes':1, "timeout":120, 'source':source, "user":user, 'ip':self.ipAddress, 'port':12345} rest.update_autofailover_settings(False, expectedResults['timeout']) elif (ops == 'resetAutoFailover'): expectedResults = {'source':source, "user":user, 'ip':self.ipAddress, 'port':12345} rest.reset_autofailover() elif (ops == 'enableClusterAlerts'): expectedResults = {"encrypt":False, "email_server:port":25, "host":"localhost", "email_server:user":"******", "alerts":["auto_failover_node", "auto_failover_maximum_reached"], \ "recipients":["*****@*****.**"], "sender":"*****@*****.**", "source":"ns_server", "user":"******", 'ip':self.ipAddress, 'port':1234} rest.set_alerts_settings('*****@*****.**', '*****@*****.**', 'ritam', 'password',) elif (ops == 'disableClusterAlerts'): rest.set_alerts_settings('*****@*****.**', '*****@*****.**', 'ritam', 'password',) expectedResults = {'source':source, "user":user, 'ip':self.ipAddress, 'port':1234} rest.disable_alerts() elif (ops == 'modifyCompactionSettingsPercentage'): expectedResults = {"parallel_db_and_view_compaction":False, "database_fragmentation_threshold:percentage":50, "view_fragmentation_threshold:percentage":50, "purge_interval":3, "source":"ns_server", "user":"******", 'source':source, "user":user, 'ip':self.ipAddress, 'port':1234} rest.set_auto_compaction(dbFragmentThresholdPercentage=50, viewFragmntThresholdPercentage=50) elif (ops == 'modifyCompactionSettingsPercentSize'): expectedResults = {"parallel_db_and_view_compaction":False, "database_fragmentation_threshold:percentage":50, "database_fragmentation_threshold:size":10, "view_fragmentation_threshold:percentage":50, "view_fragmentation_threshold:size":10, "purge_interval":3, "source":"ns_server", "user":"******", 'source':source, "user":user, 'ip':self.ipAddress, 'port':1234} rest.set_auto_compaction(dbFragmentThresholdPercentage=50, viewFragmntThresholdPercentage=50, dbFragmentThreshold=10, viewFragmntThreshold=10) elif (ops == 'modifyCompactionSettingsTime'): expectedResults = {"parallel_db_and_view_compaction":False, "database_fragmentation_threshold:percentage":50, "database_fragmentation_threshold:size":10, "view_fragmentation_threshold:percentage":50, "view_fragmentation_threshold:size":10, "allowed_time_period:abort_outside":True, "allowed_time_period:to_minute":15, "allowed_time_period:from_minute":12, "allowed_time_period:to_hour":1, "allowed_time_period:from_hour":1, "purge_interval":3, "source":"ns_server", "user":"******", 'source':source, "user":user, 'ip':self.ipAddress, 'port':1234, } rest.set_auto_compaction(dbFragmentThresholdPercentage=50, viewFragmntThresholdPercentage=50, dbFragmentThreshold=10, viewFragmntThreshold=10, allowedTimePeriodFromHour=1, allowedTimePeriodFromMin=12, allowedTimePeriodToHour=1, allowedTimePeriodToMin=15, allowedTimePeriodAbort='true') elif (ops == "AddGroup"): expectedResults = {'group_name':'add group', 'source':source, 'user':user, 'ip':self.ipAddress, 'port':1234} rest.add_zone(expectedResults['group_name']) tempStr = rest.get_zone_uri()[expectedResults['group_name']] tempStr = (tempStr.split("/"))[4] expectedResults['uuid'] = tempStr elif (ops == "UpdateGroup"): expectedResults = {'group_name':'upGroup', 'source':source, 'user':user, 'ip':self.ipAddress, 'port':1234, 'nodes':[]} rest.add_zone(expectedResults['group_name']) rest.rename_zone(expectedResults['group_name'], 'update group') expectedResults['group_name'] = 'update group' tempStr = rest.get_zone_uri()[expectedResults['group_name']] tempStr = (tempStr.split("/"))[4] expectedResults['uuid'] = tempStr elif (ops == "UpdateGroupAddNodes"): sourceGroup = "Group 1" destGroup = 'destGroup' expectedResults = {'group_name':destGroup, 'source':source, 'user':user, 'ip':self.ipAddress, 'port':1234, 'nodes':['ns_1@' + self.master.ip], 'port':1234} #rest.add_zone(sourceGroup) rest.add_zone(destGroup) self.sleep(30) rest.shuffle_nodes_in_zones([self.master.ip], sourceGroup, destGroup) tempStr = rest.get_zone_uri()[expectedResults['group_name']] tempStr = (tempStr.split("/"))[4] expectedResults['uuid'] = tempStr elif (ops == "DeleteGroup"): expectedResults = {'group_name':'delete group', 'source':source, 'user':user, 'ip':self.ipAddress, 'port':1234} rest.add_zone(expectedResults['group_name']) tempStr = rest.get_zone_uri()[expectedResults['group_name']] tempStr = (tempStr.split("/"))[4] expectedResults['uuid'] = tempStr rest.delete_zone(expectedResults['group_name']) elif (ops == "regenCer"): expectedResults = {'source':source, 'user':user, 'ip':self.ipAddress, 'port':1234} rest.regenerate_cluster_certificate() elif (ops == 'renameNode'): rest.rename_node(self.master.ip, user, password) expectedResults = {"hostname":self.master.ip, "node":"ns_1@" + self.master.ip, "source":source, "user":user, "ip":self.ipAddress, "port":56845} try: self.checkConfig(self.eventID, self.master, expectedResults) finally: if (ops == "UpdateGroupAddNodes"): sourceGroup = "Group 1" destGroup = 'destGroup' rest.shuffle_nodes_in_zones([self.master.ip], destGroup, sourceGroup) rest = RestConnection(self.master) zones = rest.get_zone_names() for zone in zones: if zone != "Group 1": rest.delete_zone(zone)