def check(self):
# check bucket compaction status across all nodes
nodes = self.rest.get_nodes()
current_compaction_count = {}
for node in nodes:
current_compaction_count[node.ip] = 0
s = TestInputServer()
s.ip = node.ip
s.ssh_username = self.server.ssh_username
s.ssh_password = self.server.ssh_password
shell = RemoteMachineShellConnection(s)
res = Cbstats(shell).get_kvtimings()
shell.disconnect()
for i in res[0]:
# check for lines that look like
# rw_0:compact_131072,262144: 8
if 'compact' in i:
current_compaction_count[node.ip] += int(i.split(':')[2])
if cmp(current_compaction_count, self.compaction_count) == 1:
# compaction count has increased
self.set_result(True)
self.state = FINISHED
else:
if self.retries > 0:
# retry
self.retries = self.retries - 1
self.task_manager.schedule(self, 10)
else:
# never detected a compaction task running
self.set_result(False)
self.state = FINISHED
def get_nodes_from_services_map(self, service_type="n1ql",
get_all_nodes=False, servers=None,
master=None):
if not servers:
servers = self.cluster.servers
if not master:
master = self.cluster.master
services_map = self.get_services_map(master=master)
if service_type not in services_map:
self.log.warning("Cannot find service node {0} in cluster "
.format(service_type))
else:
node_list = []
for server_info in services_map[service_type]:
tokens = server_info.rsplit(":", 1)
ip = tokens[0]
port = int(tokens[1])
for server in servers:
""" In tests use hostname, if IP in ini file use IP, we
need to convert it to hostname to compare it with
hostname in cluster """
if "couchbase.com" in ip and "couchbase.com" not in server.ip:
shell = RemoteMachineShellConnection(server)
hostname = shell.get_full_hostname()
self.log.debug("convert IP: {0} to hostname: {1}"
.format(server.ip, hostname))
server.ip = hostname
shell.disconnect()
elif "couchbase.com" in server.ip and "couchbase.com" not in ip:
node = TestInputServer()
node.ip = ip
""" match node.ip to server in ini file to get correct credential """
for server in servers:
shell = RemoteMachineShellConnection(server)
ips = shell.get_ip_address()
ips_new = []
for ele in ips:
ele = ele.replace('\n', '')
ips_new.append(ele)
if node.ip in ips_new:
node.ssh_username = server.ssh_username
node.ssh_password = server.ssh_password
break
shell = RemoteMachineShellConnection(node)
hostname = shell.get_full_hostname()
self.log.info("convert IP: {0} to hostname: {1}" \
.format(ip, hostname))
ip = hostname
shell.disconnect()
if (port != constants.port and port == int(server.port)) \
or (port == constants.port and server.ip == ip):
node_list.append(server)
self.log.debug("All nodes in cluster: {0}".format(node_list))
if get_all_nodes:
return node_list
else:
return node_list[0]
def main():
print('in main')
usage = '%prog -i inifile -o outputfile -s servers'
parser = OptionParser(usage)
parser.add_option('-s', '--servers', dest='servers')
parser.add_option('-x',
'--internal_servers',
dest='internal_servers',
default=None)
parser.add_option('-d',
'--addPoolServerId',
dest='addPoolServerId',
default=None)
parser.add_option('-a',
'--addPoolServers',
dest='addPoolServers',
default=None)
parser.add_option('-i', '--inifile', dest='inifile')
parser.add_option('-o', '--outputFile', dest='outputFile')
parser.add_option('-p', '--os', dest='os')
parser.add_option('-k', '--keyValue', dest='keyValue')
parser.add_option('-r', '--replaceValue', dest='replaceValue')
parser.add_option('-m',
'--skip_mem_info',
dest='skip_mem_info',
action='store_true',
default=False)
options, args = parser.parse_args()
print('the ini file is', options.inifile)
servers = []
DEFAULT_LINUX_USER = '******'
DEFAULT_LINUX_PWD = 'couchbase'
DEFAULT_WIN_USER = '******'
DEFAULT_WIN_PWD = 'Membase123'
print('the given server info is', options.servers)
if options.servers:
if not options.servers.startswith('['):
options.servers = '[' + options.servers + ']'
if options.internal_servers and not options.internal_servers.startswith(
'['):
options.internal_servers = '[' + options.internal_servers + ']'
servers = json.loads(options.servers)
internal_servers = json.loads(
options.internal_servers) if options.internal_servers else None
internal_servers_map = {}
# Sort servers by total memory
test_servers = []
for i, server_ip in enumerate(servers):
server = TestInputServer()
server.ip = server_ip
if internal_servers:
server.internal_ip = internal_servers[i]
internal_servers_map[server.ip] = server.internal_ip
server.os = options.os
if 'windows' in options.os:
server.ssh_username = DEFAULT_WIN_USER
server.ssh_password = DEFAULT_WIN_PWD
else:
if options.inifile:
with open(options.inifile, 'rt') as tempfile:
for line in tempfile:
if line.startswith('username:'******':')[1].strip()
elif line.startswith('password:'******':')[1].strip()
if server.ssh_username and server.ssh_password:
break
if not server.ssh_username:
server.ssh_username = DEFAULT_LINUX_USER
if not server.ssh_password:
server.ssh_password = DEFAULT_LINUX_PWD
test_servers.append(server)
if not options.skip_mem_info:
runner = memInfoRunner(test_servers)
runner.run()
orig_servers = servers
servers = []
if len(runner.succ) > 0:
sorted_by_mem = sorted(runner.succ.items(),
key=lambda item: int(item[1]))
print('the servers memory info is', sorted_by_mem)
for (k, v) in sorted_by_mem:
servers.append(k)
for (server, e) in runner.fail:
print("CAN'T GET MEMORY FROM {0}: {1}".format(server, e))
servers.append(server)
for nomemserver in orig_servers:
if nomemserver not in servers:
print("CAN'T GET MEMORY FROM {0}: unknown error".format(
server))
servers.append(nomemserver)
addPoolServers = []
if options.addPoolServers != None and options.addPoolServers != "None":
if not options.addPoolServers.startswith('['):
options.addPoolServers = '[' + options.addPoolServers + ']'
print('the additional server pool info is', options.addPoolServers)
addPoolServers = json.loads(options.addPoolServers)
if options.keyValue:
if options.outputFile:
update_config(options.inifile, options.keyValue,
options.outputFile)
else:
update_config(options.inifile, options.keyValue, None)
if options.keyValue and options.outputFile:
f = open(options.outputFile)
else:
f = open(options.inifile)
data = f.readlines()
for i in range(len(data)):
if 'dynamic' in data[i] and servers:
ip = servers[0]
data[i] = data[i].replace('dynamic', ip)
servers.pop(0)
if internal_servers_map:
data[i] += f"internal_ip:{internal_servers_map[ip]}\n"
elif addPoolServers:
if options.addPoolServerId == "localstack" and "endpoint:" in data[
i]:
endpoint = data[i].split(":", 1)[1]
data[i] = data[i].replace(
endpoint, "http://" + addPoolServers[0] + ":4572\n")
addPoolServers.pop(0)
elif options.addPoolServerId in data[i]:
data[i] = data[i].replace(options.addPoolServerId,
addPoolServers[0])
addPoolServers.pop(0)
if 'windows' in options.os:
if 'username:root' in data[i]:
data[i] = data[i].replace('root', DEFAULT_WIN_USER)
if 'password:couchbase' in data[i]:
data[i] = data[i].replace('couchbase', DEFAULT_WIN_PWD)
if 'es_ssh_username:root' in data[i]:
data[i] = data[i].replace('es_ssh_username:root',
'username:'******'es_ssh_password:couchbase' in data[i]:
data[i] = data[i].replace('es_ssh_password:couchbase',
'password:'******'es_ssh_username:Administrator' in data[i]:
data[i] = data[i].replace('es_ssh_username:Administrator',
'username:'******'es_ssh_password:Membase123' in data[i]:
data[i] = data[i].replace('es_ssh_password:Membase123',
'password:'******','):
old, new = oldnew.split("=")
if old in data[i]:
data[i] = data[i].replace(old, new)
for d in data:
print(d.strip())
if options.outputFile:
f = open(options.outputFile, 'w')
f.writelines(data)
else:
for d in data:
print(d.strip())
def get_nodes_from_services_map(self,
service_type="n1ql",
get_all_nodes=False,
servers=None,
master=None):
if not servers:
servers = self.servers
if not master:
master = self.master
self.get_services_map(master=master)
if (service_type not in self.services_map):
log.info("cannot find service node {0} in cluster " \
.format(service_type))
else:
list = []
for server_info in self.services_map[service_type]:
tokens = server_info.rsplit(":", 1)
ip = tokens[0]
port = int(tokens[1])
for server in servers:
""" In tests use hostname, if IP in ini file use IP, we need
to convert it to hostname to compare it with hostname
in cluster """
if "couchbase.com" in ip and "couchbase.com" not in server.ip:
shell = RemoteMachineShellConnection(server)
hostname = shell.get_full_hostname()
log.info("convert IP: {0} to hostname: {1}" \
.format(server.ip, hostname))
server.ip = hostname
shell.disconnect()
elif ip.endswith(".svc"):
from kubernetes import client as kubeClient, config as kubeConfig
currNamespace = ip.split('.')[2]
kubeConfig.load_incluster_config()
v1 = kubeClient.CoreV1Api()
nodeList = v1.list_pod_for_all_namespaces(watch=False)
for node in nodeList.items:
if node.metadata.namespace == currNamespace and \
node.status.pod_ip == server.ip:
ip = node.status.pod_ip
break
elif "couchbase.com" in server.ip and "couchbase.com" not in ip:
node = TestInputServer()
node.ip = ip
""" match node.ip to server in ini file to get correct credential """
for server in servers:
shell = RemoteMachineShellConnection(server)
ips = shell.get_ip_address()
if node.ip in ips:
node.ssh_username = server.ssh_username
node.ssh_password = server.ssh_password
break
shell = RemoteMachineShellConnection(node)
hostname = shell.get_full_hostname()
log.info("convert IP: {0} to hostname: {1}" \
.format(ip, hostname))
ip = hostname
shell.disconnect()
if (port != 8091 and port == int(server.port)) or \
(port == 8091 and server.ip.lower() == ip.lower()):
list.append(server)
log.info("list of {0} nodes in cluster: {1}".format(
service_type, list))
if get_all_nodes:
return list
else:
try:
if len(list) == 0:
list.append(servers[0])
return list[0]
except IndexError as e:
log.info(self.services_map)
raise e
