def check(self): # check bucket compaction status across all nodes nodes = self.rest.get_nodes() current_compaction_count = {} for node in nodes: current_compaction_count[node.ip] = 0 s = TestInputServer() s.ip = node.ip s.ssh_username = self.server.ssh_username s.ssh_password = self.server.ssh_password shell = RemoteMachineShellConnection(s) res = Cbstats(shell).get_kvtimings() shell.disconnect() for i in res[0]: # check for lines that look like # rw_0:compact_131072,262144: 8 if 'compact' in i: current_compaction_count[node.ip] += int(i.split(':')[2]) if cmp(current_compaction_count, self.compaction_count) == 1: # compaction count has increased self.set_result(True) self.state = FINISHED else: if self.retries > 0: # retry self.retries = self.retries - 1 self.task_manager.schedule(self, 10) else: # never detected a compaction task running self.set_result(False) self.state = FINISHED
def get_nodes_from_services_map(self, service_type="n1ql", get_all_nodes=False, servers=None, master=None): if not servers: servers = self.cluster.servers if not master: master = self.cluster.master services_map = self.get_services_map(master=master) if service_type not in services_map: self.log.warning("Cannot find service node {0} in cluster " .format(service_type)) else: node_list = [] for server_info in services_map[service_type]: tokens = server_info.rsplit(":", 1) ip = tokens[0] port = int(tokens[1]) for server in servers: """ In tests use hostname, if IP in ini file use IP, we need to convert it to hostname to compare it with hostname in cluster """ if "couchbase.com" in ip and "couchbase.com" not in server.ip: shell = RemoteMachineShellConnection(server) hostname = shell.get_full_hostname() self.log.debug("convert IP: {0} to hostname: {1}" .format(server.ip, hostname)) server.ip = hostname shell.disconnect() elif "couchbase.com" in server.ip and "couchbase.com" not in ip: node = TestInputServer() node.ip = ip """ match node.ip to server in ini file to get correct credential """ for server in servers: shell = RemoteMachineShellConnection(server) ips = shell.get_ip_address() ips_new = [] for ele in ips: ele = ele.replace('\n', '') ips_new.append(ele) if node.ip in ips_new: node.ssh_username = server.ssh_username node.ssh_password = server.ssh_password break shell = RemoteMachineShellConnection(node) hostname = shell.get_full_hostname() self.log.info("convert IP: {0} to hostname: {1}" \ .format(ip, hostname)) ip = hostname shell.disconnect() if (port != constants.port and port == int(server.port)) \ or (port == constants.port and server.ip == ip): node_list.append(server) self.log.debug("All nodes in cluster: {0}".format(node_list)) if get_all_nodes: return node_list else: return node_list[0]
def main(): print('in main') usage = '%prog -i inifile -o outputfile -s servers' parser = OptionParser(usage) parser.add_option('-s', '--servers', dest='servers') parser.add_option('-x', '--internal_servers', dest='internal_servers', default=None) parser.add_option('-d', '--addPoolServerId', dest='addPoolServerId', default=None) parser.add_option('-a', '--addPoolServers', dest='addPoolServers', default=None) parser.add_option('-i', '--inifile', dest='inifile') parser.add_option('-o', '--outputFile', dest='outputFile') parser.add_option('-p', '--os', dest='os') parser.add_option('-k', '--keyValue', dest='keyValue') parser.add_option('-r', '--replaceValue', dest='replaceValue') parser.add_option('-m', '--skip_mem_info', dest='skip_mem_info', action='store_true', default=False) options, args = parser.parse_args() print('the ini file is', options.inifile) servers = [] DEFAULT_LINUX_USER = '******' DEFAULT_LINUX_PWD = 'couchbase' DEFAULT_WIN_USER = '******' DEFAULT_WIN_PWD = 'Membase123' print('the given server info is', options.servers) if options.servers: if not options.servers.startswith('['): options.servers = '[' + options.servers + ']' if options.internal_servers and not options.internal_servers.startswith( '['): options.internal_servers = '[' + options.internal_servers + ']' servers = json.loads(options.servers) internal_servers = json.loads( options.internal_servers) if options.internal_servers else None internal_servers_map = {} # Sort servers by total memory test_servers = [] for i, server_ip in enumerate(servers): server = TestInputServer() server.ip = server_ip if internal_servers: server.internal_ip = internal_servers[i] internal_servers_map[server.ip] = server.internal_ip server.os = options.os if 'windows' in options.os: server.ssh_username = DEFAULT_WIN_USER server.ssh_password = DEFAULT_WIN_PWD else: if options.inifile: with open(options.inifile, 'rt') as tempfile: for line in tempfile: if line.startswith('username:'******':')[1].strip() elif line.startswith('password:'******':')[1].strip() if server.ssh_username and server.ssh_password: break if not server.ssh_username: server.ssh_username = DEFAULT_LINUX_USER if not server.ssh_password: server.ssh_password = DEFAULT_LINUX_PWD test_servers.append(server) if not options.skip_mem_info: runner = memInfoRunner(test_servers) runner.run() orig_servers = servers servers = [] if len(runner.succ) > 0: sorted_by_mem = sorted(runner.succ.items(), key=lambda item: int(item[1])) print('the servers memory info is', sorted_by_mem) for (k, v) in sorted_by_mem: servers.append(k) for (server, e) in runner.fail: print("CAN'T GET MEMORY FROM {0}: {1}".format(server, e)) servers.append(server) for nomemserver in orig_servers: if nomemserver not in servers: print("CAN'T GET MEMORY FROM {0}: unknown error".format( server)) servers.append(nomemserver) addPoolServers = [] if options.addPoolServers != None and options.addPoolServers != "None": if not options.addPoolServers.startswith('['): options.addPoolServers = '[' + options.addPoolServers + ']' print('the additional server pool info is', options.addPoolServers) addPoolServers = json.loads(options.addPoolServers) if options.keyValue: if options.outputFile: update_config(options.inifile, options.keyValue, options.outputFile) else: update_config(options.inifile, options.keyValue, None) if options.keyValue and options.outputFile: f = open(options.outputFile) else: f = open(options.inifile) data = f.readlines() for i in range(len(data)): if 'dynamic' in data[i] and servers: ip = servers[0] data[i] = data[i].replace('dynamic', ip) servers.pop(0) if internal_servers_map: data[i] += f"internal_ip:{internal_servers_map[ip]}\n" elif addPoolServers: if options.addPoolServerId == "localstack" and "endpoint:" in data[ i]: endpoint = data[i].split(":", 1)[1] data[i] = data[i].replace( endpoint, "http://" + addPoolServers[0] + ":4572\n") addPoolServers.pop(0) elif options.addPoolServerId in data[i]: data[i] = data[i].replace(options.addPoolServerId, addPoolServers[0]) addPoolServers.pop(0) if 'windows' in options.os: if 'username:root' in data[i]: data[i] = data[i].replace('root', DEFAULT_WIN_USER) if 'password:couchbase' in data[i]: data[i] = data[i].replace('couchbase', DEFAULT_WIN_PWD) if 'es_ssh_username:root' in data[i]: data[i] = data[i].replace('es_ssh_username:root', 'username:'******'es_ssh_password:couchbase' in data[i]: data[i] = data[i].replace('es_ssh_password:couchbase', 'password:'******'es_ssh_username:Administrator' in data[i]: data[i] = data[i].replace('es_ssh_username:Administrator', 'username:'******'es_ssh_password:Membase123' in data[i]: data[i] = data[i].replace('es_ssh_password:Membase123', 'password:'******','): old, new = oldnew.split("=") if old in data[i]: data[i] = data[i].replace(old, new) for d in data: print(d.strip()) if options.outputFile: f = open(options.outputFile, 'w') f.writelines(data) else: for d in data: print(d.strip())
def get_nodes_from_services_map(self, service_type="n1ql", get_all_nodes=False, servers=None, master=None): if not servers: servers = self.servers if not master: master = self.master self.get_services_map(master=master) if (service_type not in self.services_map): log.info("cannot find service node {0} in cluster " \ .format(service_type)) else: list = [] for server_info in self.services_map[service_type]: tokens = server_info.rsplit(":", 1) ip = tokens[0] port = int(tokens[1]) for server in servers: """ In tests use hostname, if IP in ini file use IP, we need to convert it to hostname to compare it with hostname in cluster """ if "couchbase.com" in ip and "couchbase.com" not in server.ip: shell = RemoteMachineShellConnection(server) hostname = shell.get_full_hostname() log.info("convert IP: {0} to hostname: {1}" \ .format(server.ip, hostname)) server.ip = hostname shell.disconnect() elif ip.endswith(".svc"): from kubernetes import client as kubeClient, config as kubeConfig currNamespace = ip.split('.')[2] kubeConfig.load_incluster_config() v1 = kubeClient.CoreV1Api() nodeList = v1.list_pod_for_all_namespaces(watch=False) for node in nodeList.items: if node.metadata.namespace == currNamespace and \ node.status.pod_ip == server.ip: ip = node.status.pod_ip break elif "couchbase.com" in server.ip and "couchbase.com" not in ip: node = TestInputServer() node.ip = ip """ match node.ip to server in ini file to get correct credential """ for server in servers: shell = RemoteMachineShellConnection(server) ips = shell.get_ip_address() if node.ip in ips: node.ssh_username = server.ssh_username node.ssh_password = server.ssh_password break shell = RemoteMachineShellConnection(node) hostname = shell.get_full_hostname() log.info("convert IP: {0} to hostname: {1}" \ .format(ip, hostname)) ip = hostname shell.disconnect() if (port != 8091 and port == int(server.port)) or \ (port == 8091 and server.ip.lower() == ip.lower()): list.append(server) log.info("list of {0} nodes in cluster: {1}".format( service_type, list)) if get_all_nodes: return list else: try: if len(list) == 0: list.append(servers[0]) return list[0] except IndexError as e: log.info(self.services_map) raise e