def get_nodes_from_services_map(self, service_type="n1ql",
get_all_nodes=False, servers=None,
master=None):
if not servers:
servers = self.cluster.servers
if not master:
master = self.cluster.master
services_map = self.get_services_map(master=master)
if service_type not in services_map:
self.log.warning("Cannot find service node {0} in cluster "
.format(service_type))
else:
node_list = []
for server_info in services_map[service_type]:
tokens = server_info.rsplit(":", 1)
ip = tokens[0]
port = int(tokens[1])
for server in servers:
""" In tests use hostname, if IP in ini file use IP, we
need to convert it to hostname to compare it with
hostname in cluster """
if "couchbase.com" in ip and "couchbase.com" not in server.ip:
shell = RemoteMachineShellConnection(server)
hostname = shell.get_full_hostname()
self.log.debug("convert IP: {0} to hostname: {1}"
.format(server.ip, hostname))
server.ip = hostname
shell.disconnect()
elif "couchbase.com" in server.ip and "couchbase.com" not in ip:
node = TestInputServer()
node.ip = ip
""" match node.ip to server in ini file to get correct credential """
for server in servers:
shell = RemoteMachineShellConnection(server)
ips = shell.get_ip_address()
ips_new = []
for ele in ips:
ele = ele.replace('\n', '')
ips_new.append(ele)
if node.ip in ips_new:
node.ssh_username = server.ssh_username
node.ssh_password = server.ssh_password
break
shell = RemoteMachineShellConnection(node)
hostname = shell.get_full_hostname()
self.log.info("convert IP: {0} to hostname: {1}" \
.format(ip, hostname))
ip = hostname
shell.disconnect()
if (port != constants.port and port == int(server.port)) \
or (port == constants.port and server.ip == ip):
node_list.append(server)
self.log.debug("All nodes in cluster: {0}".format(node_list))
if get_all_nodes:
return node_list
else:
return node_list[0]
def get_internal_IP(self, server):
shell = RemoteMachineShellConnection(server)
internal_IP = shell.get_ip_address()
internal_IP = [x for x in internal_IP if x != "127.0.0.1"]
shell.disconnect()
if internal_IP:
return internal_IP[0]
else:
self.fail("Fail to get internal IP")
def cleanup_cluster(servers, wait_for_rebalance=True, master=None):
log = logger.Logger.get_logger()
if master is None:
master = servers[0]
rest = RestConnection(master)
helper = RestHelper(rest)
helper.is_ns_server_running(
timeout_in_seconds=testconstants.NS_SERVER_TIMEOUT)
nodes = rest.node_statuses()
master_id = rest.get_nodes_self().id
for node in nodes:
if int(node.port) in range(9091, 9991):
rest.eject_node(node)
nodes.remove(node)
if len(nodes) > 1:
log.info("rebalancing all nodes in order to remove nodes")
rest.log_client_error("Starting rebalance from test, ejected nodes %s" % \
[node.id for node in nodes if node.id != master_id])
removed = helper.remove_nodes(
knownNodes=[node.id for node in nodes],
ejectedNodes=[
node.id for node in nodes if node.id != master_id
],
wait_for_rebalance=wait_for_rebalance)
success_cleaned = []
alt_addr = TestInputSingleton.input.param("alt_addr", False)
for removed in [node for node in nodes if (node.id != master_id)]:
removed.rest_password = servers[0].rest_password
removed.rest_username = servers[0].rest_username
try:
if alt_addr:
for server in servers:
shell = RemoteMachineShellConnection(server)
internal_IP = shell.get_ip_address()
internal_IP = [
x for x in internal_IP if x != "127.0.0.1"
]
shell.disconnect()
if internal_IP == removed.ip:
rest = RestConnection(server)
break
else:
rest = RestConnection(removed)
except Exception as ex:
log.error(
"can't create rest connection after rebalance out for ejected nodes,\
will retry after 10 seconds according to MB-8430: {0} "
.format(ex))
time.sleep(10)
rest = RestConnection(removed)
start = time.time()
while time.time() - start < 30:
if len(rest.get_pools_info()["pools"]) == 0:
success_cleaned.append(removed)
break
else:
time.sleep(0.1)
if time.time() - start > 10:
log.error("'pools' on node {0}:{1} - {2}".format(
removed.ip, removed.port,
rest.get_pools_info()["pools"]))
for node in {node
for node in nodes
if (node.id != master_id)} - set(success_cleaned):
log.error(
"node {0}:{1} was not cleaned after removing from cluster".
format(removed.ip, removed.port))
try:
if alt_addr:
for server in servers:
shell = RemoteMachineShellConnection(server)
internal_IP = shell.get_ip_address()
internal_IP = [
x for x in internal_IP if x != "127.0.0.1"
]
shell.disconnect()
if internal_IP == removed.ip:
rest = RestConnection(server)
break
else:
rest = RestConnection(node)
if not alt_addr:
rest.force_eject_node()
except Exception as ex:
log.error("force_eject_node {0}:{1} failed: {2}".format(
removed.ip, removed.port, ex))
if len({node for node in nodes if (node.id != master_id)}\
- set(success_cleaned)) != 0:
if not alt_addr:
raise Exception(
"not all ejected nodes were cleaned successfully")
log.info("removed all the nodes from cluster associated with {0} ? {1}".format(servers[0], \
[(node.id, node.port) for node in nodes if (node.id != master_id)]))
def get_nodes_from_services_map(self,
service_type="n1ql",
get_all_nodes=False,
servers=None,
master=None):
if not servers:
servers = self.servers
if not master:
master = self.master
self.get_services_map(master=master)
if (service_type not in self.services_map):
log.info("cannot find service node {0} in cluster " \
.format(service_type))
else:
list = []
for server_info in self.services_map[service_type]:
tokens = server_info.rsplit(":", 1)
ip = tokens[0]
port = int(tokens[1])
for server in servers:
""" In tests use hostname, if IP in ini file use IP, we need
to convert it to hostname to compare it with hostname
in cluster """
if "couchbase.com" in ip and "couchbase.com" not in server.ip:
shell = RemoteMachineShellConnection(server)
hostname = shell.get_full_hostname()
log.info("convert IP: {0} to hostname: {1}" \
.format(server.ip, hostname))
server.ip = hostname
shell.disconnect()
elif ip.endswith(".svc"):
from kubernetes import client as kubeClient, config as kubeConfig
currNamespace = ip.split('.')[2]
kubeConfig.load_incluster_config()
v1 = kubeClient.CoreV1Api()
nodeList = v1.list_pod_for_all_namespaces(watch=False)
for node in nodeList.items:
if node.metadata.namespace == currNamespace and \
node.status.pod_ip == server.ip:
ip = node.status.pod_ip
break
elif "couchbase.com" in server.ip and "couchbase.com" not in ip:
node = TestInputServer()
node.ip = ip
""" match node.ip to server in ini file to get correct credential """
for server in servers:
shell = RemoteMachineShellConnection(server)
ips = shell.get_ip_address()
if node.ip in ips:
node.ssh_username = server.ssh_username
node.ssh_password = server.ssh_password
break
shell = RemoteMachineShellConnection(node)
hostname = shell.get_full_hostname()
log.info("convert IP: {0} to hostname: {1}" \
.format(ip, hostname))
ip = hostname
shell.disconnect()
if (port != 8091 and port == int(server.port)) or \
(port == 8091 and server.ip.lower() == ip.lower()):
list.append(server)
log.info("list of {0} nodes in cluster: {1}".format(
service_type, list))
if get_all_nodes:
return list
else:
try:
if len(list) == 0:
list.append(servers[0])
return list[0]
except IndexError as e:
log.info(self.services_map)
raise e
