def get_nodes_formatted(pod,
tenant,
cluster_id,
username=None,
password=None):
servers = capella_utils.get_nodes(pod, tenant, cluster_id)
nodes = list()
for server in servers:
temp_server = TestInputServer()
temp_server.ip = server.get("hostname")
temp_server.hostname = server.get("hostname")
capella_services = server.get("services")
services = []
for service in capella_services:
if service == "Data":
services.append("kv")
elif service == "Index":
services.append("index")
elif service == "Query":
services.append("n1ql")
elif service == "Search":
services.append("fts")
elif service == "Eventing":
services.append("eventing")
elif service == "Analytics":
services.append("cbas")
temp_server.services = services
temp_server.port = "18091"
temp_server.rest_username = username
temp_server.rest_password = password
temp_server.hosted_on_cloud = True
temp_server.memcached_port = "11207"
nodes.append(temp_server)
return nodes
def reset_vbucket(self, rest, key):
vBucketId = crc32.crc32_hash(key) & (len(self.vBucketMap) - 1)
forward_map = rest.get_bucket(self.bucket).forward_map
if not forward_map:
forward_map = rest.get_vbuckets(self.bucket)
nodes = rest.get_nodes()
for vBucket in forward_map:
if vBucketId == vBucket.id:
self.vBucketMap[vBucket.id] = vBucket.master
# it has changed , then to different server or a new server
masterIp = vBucket.master.split(":")[0]
masterPort = int(vBucket.master.split(":")[1])
if self.vBucketMap[vBucketId] not in self.memcacheds:
server = TestInputServer()
server.rest_username = rest.username
server.rest_password = rest.password
for node in nodes:
if node.ip == masterIp and node.memcached == masterPort:
server.port = node.port
server.ip = masterIp
self.log.info("Recevied forward map, reset vbucket map, new direct_client")
self.memcacheds[vBucket.master] = MemcachedClientHelper.direct_client(server, self.bucket)
return True
else:
# if no one is using that memcached connection anymore just close the connection
return True
return False
def check(self):
# check bucket compaction status across all nodes
nodes = self.rest.get_nodes()
current_compaction_count = {}
for node in nodes:
current_compaction_count[node.ip] = 0
s = TestInputServer()
s.ip = node.ip
s.ssh_username = self.server.ssh_username
s.ssh_password = self.server.ssh_password
shell = RemoteMachineShellConnection(s)
res = Cbstats(shell).get_kvtimings()
shell.disconnect()
for i in res[0]:
# check for lines that look like
# rw_0:compact_131072,262144: 8
if 'compact' in i:
current_compaction_count[node.ip] += int(i.split(':')[2])
if cmp(current_compaction_count, self.compaction_count) == 1:
# compaction count has increased
self.set_result(True)
self.state = FINISHED
else:
if self.retries > 0:
# retry
self.retries = self.retries - 1
self.task_manager.schedule(self, 10)
else:
# never detected a compaction task running
self.set_result(False)
self.state = FINISHED
def _poxi(self):
tServer = TestInputServer()
tServer.ip = self.server_ip
tServer.rest_username = "******"
tServer.rest_password = "******"
tServer.port = 8091
rest = RestConnection(tServer)
return VBucketAwareMemcached(rest, self.bucket_name)
def _poxi(self):
tServer = TestInputServer()
tServer.ip = self.server_ip
tServer.rest_username = "******"
tServer.rest_password = "******"
tServer.port = 8091
rest = RestConnection(tServer)
return VBucketAwareMemcached(rest, self.bucket_name)
def get_nodes_from_services_map(self, service_type="n1ql",
get_all_nodes=False, servers=None,
master=None):
if not servers:
servers = self.cluster.servers
if not master:
master = self.cluster.master
services_map = self.get_services_map(master=master)
if service_type not in services_map:
self.log.warning("Cannot find service node {0} in cluster "
.format(service_type))
else:
node_list = []
for server_info in services_map[service_type]:
tokens = server_info.rsplit(":", 1)
ip = tokens[0]
port = int(tokens[1])
for server in servers:
""" In tests use hostname, if IP in ini file use IP, we
need to convert it to hostname to compare it with
hostname in cluster """
if "couchbase.com" in ip and "couchbase.com" not in server.ip:
shell = RemoteMachineShellConnection(server)
hostname = shell.get_full_hostname()
self.log.debug("convert IP: {0} to hostname: {1}"
.format(server.ip, hostname))
server.ip = hostname
shell.disconnect()
elif "couchbase.com" in server.ip and "couchbase.com" not in ip:
node = TestInputServer()
node.ip = ip
""" match node.ip to server in ini file to get correct credential """
for server in servers:
shell = RemoteMachineShellConnection(server)
ips = shell.get_ip_address()
ips_new = []
for ele in ips:
ele = ele.replace('\n', '')
ips_new.append(ele)
if node.ip in ips_new:
node.ssh_username = server.ssh_username
node.ssh_password = server.ssh_password
break
shell = RemoteMachineShellConnection(node)
hostname = shell.get_full_hostname()
self.log.info("convert IP: {0} to hostname: {1}" \
.format(ip, hostname))
ip = hostname
shell.disconnect()
if (port != constants.port and port == int(server.port)) \
or (port == constants.port and server.ip == ip):
node_list.append(server)
self.log.debug("All nodes in cluster: {0}".format(node_list))
if get_all_nodes:
return node_list
else:
return node_list[0]
def __populate_cluster_info(self, cluster_id, servers, cluster_srv,
cluster_name, service_config):
nodes = list()
for server in servers:
temp_server = TestInputServer()
temp_server.ip = server.get("hostname")
temp_server.hostname = server.get("hostname")
temp_server.services = server.get("services")
temp_server.port = "18091"
temp_server.rest_username = self.rest_username
temp_server.rest_password = self.rest_password
temp_server.hosted_on_cloud = True
temp_server.memcached_port = "11207"
nodes.append(temp_server)
cluster = CBCluster(username=self.rest_username,
password=self.rest_password,
servers=[None] * 40)
cluster.id = cluster_id
cluster.srv = cluster_srv
cluster.cluster_config = service_config
cluster.pod = self.pod
cluster.tenant = self.tenant
for temp_server in nodes:
if "Data" in temp_server.services:
cluster.kv_nodes.append(temp_server)
if "Query" in temp_server.services:
cluster.query_nodes.append(temp_server)
if "Index" in temp_server.services:
cluster.index_nodes.append(temp_server)
if "Eventing" in temp_server.services:
cluster.eventing_nodes.append(temp_server)
if "Analytics" in temp_server.services:
cluster.cbas_nodes.append(temp_server)
if "FTS" in temp_server.services:
cluster.fts_nodes.append(temp_server)
cluster.master = cluster.kv_nodes[0]
self.tenant.clusters.update({cluster.id: cluster})
cluster.master = cluster.kv_nodes[0]
self.tenant.clusters.update({cluster.id: cluster})
self.cb_clusters[cluster_name] = cluster
self.cb_clusters[cluster_name].cloud_cluster = True
def add_memcached(self, server_str, memcacheds, rest, bucket):
if not server_str in memcacheds:
serverIp = server_str.split(":")[0]
serverPort = int(server_str.split(":")[1])
nodes = rest.get_nodes()
server = TestInputServer()
server.ip = serverIp
server.port = rest.port
server.rest_username = rest.username
server.rest_password = rest.password
try:
for node in nodes:
if node.ip == serverIp and node.memcached == serverPort:
if server_str not in memcacheds:
server.port = node.port
memcacheds[server_str] = MemcachedClientHelper.direct_client(server, bucket)
break
except Exception as ex:
msg = "unable to establish connection to {0}.cleanup open connections"
self.log.warn(msg.format(serverIp))
self.done()
raise ex
def main():
print('in main')
usage = '%prog -i inifile -o outputfile -s servers'
parser = OptionParser(usage)
parser.add_option('-s', '--servers', dest='servers')
parser.add_option('-x',
'--internal_servers',
dest='internal_servers',
default=None)
parser.add_option('-d',
'--addPoolServerId',
dest='addPoolServerId',
default=None)
parser.add_option('-a',
'--addPoolServers',
dest='addPoolServers',
default=None)
parser.add_option('-i', '--inifile', dest='inifile')
parser.add_option('-o', '--outputFile', dest='outputFile')
parser.add_option('-p', '--os', dest='os')
parser.add_option('-k', '--keyValue', dest='keyValue')
parser.add_option('-r', '--replaceValue', dest='replaceValue')
parser.add_option('-m',
'--skip_mem_info',
dest='skip_mem_info',
action='store_true',
default=False)
options, args = parser.parse_args()
print('the ini file is', options.inifile)
servers = []
DEFAULT_LINUX_USER = '******'
DEFAULT_LINUX_PWD = 'couchbase'
DEFAULT_WIN_USER = '******'
DEFAULT_WIN_PWD = 'Membase123'
print('the given server info is', options.servers)
if options.servers:
if not options.servers.startswith('['):
options.servers = '[' + options.servers + ']'
if options.internal_servers and not options.internal_servers.startswith(
'['):
options.internal_servers = '[' + options.internal_servers + ']'
servers = json.loads(options.servers)
internal_servers = json.loads(
options.internal_servers) if options.internal_servers else None
internal_servers_map = {}
# Sort servers by total memory
test_servers = []
for i, server_ip in enumerate(servers):
server = TestInputServer()
server.ip = server_ip
if internal_servers:
server.internal_ip = internal_servers[i]
internal_servers_map[server.ip] = server.internal_ip
server.os = options.os
if 'windows' in options.os:
server.ssh_username = DEFAULT_WIN_USER
server.ssh_password = DEFAULT_WIN_PWD
else:
if options.inifile:
with open(options.inifile, 'rt') as tempfile:
for line in tempfile:
if line.startswith('username:'******':')[1].strip()
elif line.startswith('password:'******':')[1].strip()
if server.ssh_username and server.ssh_password:
break
if not server.ssh_username:
server.ssh_username = DEFAULT_LINUX_USER
if not server.ssh_password:
server.ssh_password = DEFAULT_LINUX_PWD
test_servers.append(server)
if not options.skip_mem_info:
runner = memInfoRunner(test_servers)
runner.run()
orig_servers = servers
servers = []
if len(runner.succ) > 0:
sorted_by_mem = sorted(runner.succ.items(),
key=lambda item: int(item[1]))
print('the servers memory info is', sorted_by_mem)
for (k, v) in sorted_by_mem:
servers.append(k)
for (server, e) in runner.fail:
print("CAN'T GET MEMORY FROM {0}: {1}".format(server, e))
servers.append(server)
for nomemserver in orig_servers:
if nomemserver not in servers:
print("CAN'T GET MEMORY FROM {0}: unknown error".format(
server))
servers.append(nomemserver)
addPoolServers = []
if options.addPoolServers != None and options.addPoolServers != "None":
if not options.addPoolServers.startswith('['):
options.addPoolServers = '[' + options.addPoolServers + ']'
print('the additional server pool info is', options.addPoolServers)
addPoolServers = json.loads(options.addPoolServers)
if options.keyValue:
if options.outputFile:
update_config(options.inifile, options.keyValue,
options.outputFile)
else:
update_config(options.inifile, options.keyValue, None)
if options.keyValue and options.outputFile:
f = open(options.outputFile)
else:
f = open(options.inifile)
data = f.readlines()
for i in range(len(data)):
if 'dynamic' in data[i] and servers:
ip = servers[0]
data[i] = data[i].replace('dynamic', ip)
servers.pop(0)
if internal_servers_map:
data[i] += f"internal_ip:{internal_servers_map[ip]}\n"
elif addPoolServers:
if options.addPoolServerId == "localstack" and "endpoint:" in data[
i]:
endpoint = data[i].split(":", 1)[1]
data[i] = data[i].replace(
endpoint, "http://" + addPoolServers[0] + ":4572\n")
addPoolServers.pop(0)
elif options.addPoolServerId in data[i]:
data[i] = data[i].replace(options.addPoolServerId,
addPoolServers[0])
addPoolServers.pop(0)
if 'windows' in options.os:
if 'username:root' in data[i]:
data[i] = data[i].replace('root', DEFAULT_WIN_USER)
if 'password:couchbase' in data[i]:
data[i] = data[i].replace('couchbase', DEFAULT_WIN_PWD)
if 'es_ssh_username:root' in data[i]:
data[i] = data[i].replace('es_ssh_username:root',
'username:'******'es_ssh_password:couchbase' in data[i]:
data[i] = data[i].replace('es_ssh_password:couchbase',
'password:'******'es_ssh_username:Administrator' in data[i]:
data[i] = data[i].replace('es_ssh_username:Administrator',
'username:'******'es_ssh_password:Membase123' in data[i]:
data[i] = data[i].replace('es_ssh_password:Membase123',
'password:'******','):
old, new = oldnew.split("=")
if old in data[i]:
data[i] = data[i].replace(old, new)
for d in data:
print(d.strip())
if options.outputFile:
f = open(options.outputFile, 'w')
f.writelines(data)
else:
for d in data:
print(d.strip())
def get_nodes_from_services_map(self,
service_type="n1ql",
get_all_nodes=False,
servers=None,
master=None):
if not servers:
servers = self.servers
if not master:
master = self.master
self.get_services_map(master=master)
if (service_type not in self.services_map):
log.info("cannot find service node {0} in cluster " \
.format(service_type))
else:
list = []
for server_info in self.services_map[service_type]:
tokens = server_info.rsplit(":", 1)
ip = tokens[0]
port = int(tokens[1])
for server in servers:
""" In tests use hostname, if IP in ini file use IP, we need
to convert it to hostname to compare it with hostname
in cluster """
if "couchbase.com" in ip and "couchbase.com" not in server.ip:
shell = RemoteMachineShellConnection(server)
hostname = shell.get_full_hostname()
log.info("convert IP: {0} to hostname: {1}" \
.format(server.ip, hostname))
server.ip = hostname
shell.disconnect()
elif ip.endswith(".svc"):
from kubernetes import client as kubeClient, config as kubeConfig
currNamespace = ip.split('.')[2]
kubeConfig.load_incluster_config()
v1 = kubeClient.CoreV1Api()
nodeList = v1.list_pod_for_all_namespaces(watch=False)
for node in nodeList.items:
if node.metadata.namespace == currNamespace and \
node.status.pod_ip == server.ip:
ip = node.status.pod_ip
break
elif "couchbase.com" in server.ip and "couchbase.com" not in ip:
node = TestInputServer()
node.ip = ip
""" match node.ip to server in ini file to get correct credential """
for server in servers:
shell = RemoteMachineShellConnection(server)
ips = shell.get_ip_address()
if node.ip in ips:
node.ssh_username = server.ssh_username
node.ssh_password = server.ssh_password
break
shell = RemoteMachineShellConnection(node)
hostname = shell.get_full_hostname()
log.info("convert IP: {0} to hostname: {1}" \
.format(ip, hostname))
ip = hostname
shell.disconnect()
if (port != 8091 and port == int(server.port)) or \
(port == 8091 and server.ip.lower() == ip.lower()):
list.append(server)
log.info("list of {0} nodes in cluster: {1}".format(
service_type, list))
if get_all_nodes:
return list
else:
try:
if len(list) == 0:
list.append(servers[0])
return list[0]
except IndexError as e:
log.info(self.services_map)
raise e