def get_nodes_from_services_map(self, service_type="n1ql", get_all_nodes=False, servers=None, master=None): if not servers: servers = self.cluster.servers if not master: master = self.cluster.master services_map = self.get_services_map(master=master) if service_type not in services_map: self.log.warning("Cannot find service node {0} in cluster " .format(service_type)) else: node_list = [] for server_info in services_map[service_type]: tokens = server_info.rsplit(":", 1) ip = tokens[0] port = int(tokens[1]) for server in servers: """ In tests use hostname, if IP in ini file use IP, we need to convert it to hostname to compare it with hostname in cluster """ if "couchbase.com" in ip and "couchbase.com" not in server.ip: shell = RemoteMachineShellConnection(server) hostname = shell.get_full_hostname() self.log.debug("convert IP: {0} to hostname: {1}" .format(server.ip, hostname)) server.ip = hostname shell.disconnect() elif "couchbase.com" in server.ip and "couchbase.com" not in ip: node = TestInputServer() node.ip = ip """ match node.ip to server in ini file to get correct credential """ for server in servers: shell = RemoteMachineShellConnection(server) ips = shell.get_ip_address() ips_new = [] for ele in ips: ele = ele.replace('\n', '') ips_new.append(ele) if node.ip in ips_new: node.ssh_username = server.ssh_username node.ssh_password = server.ssh_password break shell = RemoteMachineShellConnection(node) hostname = shell.get_full_hostname() self.log.info("convert IP: {0} to hostname: {1}" \ .format(ip, hostname)) ip = hostname shell.disconnect() if (port != constants.port and port == int(server.port)) \ or (port == constants.port and server.ip == ip): node_list.append(server) self.log.debug("All nodes in cluster: {0}".format(node_list)) if get_all_nodes: return node_list else: return node_list[0]
def get_internal_IP(self, server): shell = RemoteMachineShellConnection(server) internal_IP = shell.get_ip_address() internal_IP = [x for x in internal_IP if x != "127.0.0.1"] shell.disconnect() if internal_IP: return internal_IP[0] else: self.fail("Fail to get internal IP")
def cleanup_cluster(servers, wait_for_rebalance=True, master=None): log = logger.Logger.get_logger() if master is None: master = servers[0] rest = RestConnection(master) helper = RestHelper(rest) helper.is_ns_server_running( timeout_in_seconds=testconstants.NS_SERVER_TIMEOUT) nodes = rest.node_statuses() master_id = rest.get_nodes_self().id for node in nodes: if int(node.port) in range(9091, 9991): rest.eject_node(node) nodes.remove(node) if len(nodes) > 1: log.info("rebalancing all nodes in order to remove nodes") rest.log_client_error("Starting rebalance from test, ejected nodes %s" % \ [node.id for node in nodes if node.id != master_id]) removed = helper.remove_nodes( knownNodes=[node.id for node in nodes], ejectedNodes=[ node.id for node in nodes if node.id != master_id ], wait_for_rebalance=wait_for_rebalance) success_cleaned = [] alt_addr = TestInputSingleton.input.param("alt_addr", False) for removed in [node for node in nodes if (node.id != master_id)]: removed.rest_password = servers[0].rest_password removed.rest_username = servers[0].rest_username try: if alt_addr: for server in servers: shell = RemoteMachineShellConnection(server) internal_IP = shell.get_ip_address() internal_IP = [ x for x in internal_IP if x != "127.0.0.1" ] shell.disconnect() if internal_IP == removed.ip: rest = RestConnection(server) break else: rest = RestConnection(removed) except Exception as ex: log.error( "can't create rest connection after rebalance out for ejected nodes,\ will retry after 10 seconds according to MB-8430: {0} " .format(ex)) time.sleep(10) rest = RestConnection(removed) start = time.time() while time.time() - start < 30: if len(rest.get_pools_info()["pools"]) == 0: success_cleaned.append(removed) break else: time.sleep(0.1) if time.time() - start > 10: log.error("'pools' on node {0}:{1} - {2}".format( removed.ip, removed.port, rest.get_pools_info()["pools"])) for node in {node for node in nodes if (node.id != master_id)} - set(success_cleaned): log.error( "node {0}:{1} was not cleaned after removing from cluster". format(removed.ip, removed.port)) try: if alt_addr: for server in servers: shell = RemoteMachineShellConnection(server) internal_IP = shell.get_ip_address() internal_IP = [ x for x in internal_IP if x != "127.0.0.1" ] shell.disconnect() if internal_IP == removed.ip: rest = RestConnection(server) break else: rest = RestConnection(node) if not alt_addr: rest.force_eject_node() except Exception as ex: log.error("force_eject_node {0}:{1} failed: {2}".format( removed.ip, removed.port, ex)) if len({node for node in nodes if (node.id != master_id)}\ - set(success_cleaned)) != 0: if not alt_addr: raise Exception( "not all ejected nodes were cleaned successfully") log.info("removed all the nodes from cluster associated with {0} ? {1}".format(servers[0], \ [(node.id, node.port) for node in nodes if (node.id != master_id)]))
def get_nodes_from_services_map(self, service_type="n1ql", get_all_nodes=False, servers=None, master=None): if not servers: servers = self.servers if not master: master = self.master self.get_services_map(master=master) if (service_type not in self.services_map): log.info("cannot find service node {0} in cluster " \ .format(service_type)) else: list = [] for server_info in self.services_map[service_type]: tokens = server_info.rsplit(":", 1) ip = tokens[0] port = int(tokens[1]) for server in servers: """ In tests use hostname, if IP in ini file use IP, we need to convert it to hostname to compare it with hostname in cluster """ if "couchbase.com" in ip and "couchbase.com" not in server.ip: shell = RemoteMachineShellConnection(server) hostname = shell.get_full_hostname() log.info("convert IP: {0} to hostname: {1}" \ .format(server.ip, hostname)) server.ip = hostname shell.disconnect() elif ip.endswith(".svc"): from kubernetes import client as kubeClient, config as kubeConfig currNamespace = ip.split('.')[2] kubeConfig.load_incluster_config() v1 = kubeClient.CoreV1Api() nodeList = v1.list_pod_for_all_namespaces(watch=False) for node in nodeList.items: if node.metadata.namespace == currNamespace and \ node.status.pod_ip == server.ip: ip = node.status.pod_ip break elif "couchbase.com" in server.ip and "couchbase.com" not in ip: node = TestInputServer() node.ip = ip """ match node.ip to server in ini file to get correct credential """ for server in servers: shell = RemoteMachineShellConnection(server) ips = shell.get_ip_address() if node.ip in ips: node.ssh_username = server.ssh_username node.ssh_password = server.ssh_password break shell = RemoteMachineShellConnection(node) hostname = shell.get_full_hostname() log.info("convert IP: {0} to hostname: {1}" \ .format(ip, hostname)) ip = hostname shell.disconnect() if (port != 8091 and port == int(server.port)) or \ (port == 8091 and server.ip.lower() == ip.lower()): list.append(server) log.info("list of {0} nodes in cluster: {1}".format( service_type, list)) if get_all_nodes: return list else: try: if len(list) == 0: list.append(servers[0]) return list[0] except IndexError as e: log.info(self.services_map) raise e