def offline_cluster_upgrade_and_rebalance(self): num_stoped_nodes = self.input.param('num_stoped_nodes', self.nodes_init) stoped_nodes = self.servers[self.nodes_init - num_stoped_nodes :self.nodes_init] servs_out = self.servers[self.nodes_init - num_stoped_nodes - self.nodes_out :self.nodes_init - num_stoped_nodes] servs_in = self.servers[self.nodes_init:self.nodes_init + self.nodes_in] self._install(self.servers) self.operations(self.servers[:self.nodes_init]) if self.ddocs_num: self.create_ddocs_and_views() if self.during_ops: for opn in self.during_ops: getattr(self, opn)() for upgrade_version in self.upgrade_versions: self.sleep(self.sleep_time, "Pre-setup of old version is done. Wait for upgrade to {0} version".\ format(upgrade_version)) for server in stoped_nodes: remote = RemoteMachineShellConnection(server) remote.stop_server() remote.disconnect() upgrade_threads = self._async_update(upgrade_version, stoped_nodes) try: self.cluster.rebalance(self.servers[:self.nodes_init], servs_in, servs_out) except RebalanceFailedException: self.log.info("rebalance failed as expected") for upgrade_thread in upgrade_threads: upgrade_thread.join() success_upgrade = True while not self.queue.empty(): success_upgrade &= self.queue.get() if not success_upgrade: self.fail("Upgrade failed!") ClusterOperationHelper.wait_for_ns_servers_or_assert(stoped_nodes, self) self.cluster.rebalance(self.servers[:self.nodes_init], [], servs_out) self.dcp_rebalance_in_offline_upgrade_from_version2_to_version3() self.verification(list(set(self.servers[:self.nodes_init] + servs_in) - set(servs_out)))
def test_fileRotate20MB(self): auditIns = audit(host=self.master) firstEventTime = self.getTimeStampForFile(auditIns) tempEventCounter = 0 rest = RestConnection(self.master) shell = RemoteMachineShellConnection(self.master) filePath = auditIns.pathLogFile + auditIns.AUDITLOGFILENAME number = int (shell.get_data_file_size(filePath)) hostname = shell.execute_command("hostname") archiveFile = hostname[0][0] + '-' + firstEventTime + "-audit.log" result = shell.file_exists(auditIns.pathLogFile, archiveFile) tempTime = 0 starttime = time.time() while ((number < 21089520) and (tempTime < 36000) and (result == False)): for i in range(1, 10): status, content = rest.validateLogin("Administrator", "password", True, getContent=True) tempEventCounter += 1 number = int (shell.get_data_file_size(filePath)) currTime = time.time() tempTime = int (currTime - starttime) result = shell.file_exists(auditIns.pathLogFile, archiveFile) self.sleep(30) result = shell.file_exists(auditIns.pathLogFile, archiveFile) shell.disconnect() self.log.info ("--------Total Event Created ---- {0}".format(tempEventCounter)) self.assertTrue(result, "Archive Audit.log is not created on reaching 20MB threshhold")
def test_restart_node_with_full_disk(self): def _get_disk_usage_percentage(remote_client): disk_info = remote_client.get_disk_info() percentage = disk_info[1] + disk_info[2]; for item in percentage.split(): if "%" in item: self.log.info("disk usage {0}".format(item)) return item[:-1] remote_client = RemoteMachineShellConnection(self.master) output, error = remote_client.execute_command_raw("rm -rf full_disk*", use_channel=True) remote_client.log_command_output(output, error) percentage = _get_disk_usage_percentage(remote_client) try: while int(percentage) < 99: output, error = remote_client.execute_command("dd if=/dev/zero of=full_disk{0} bs=3G count=1".format(percentage + str(time.time())), use_channel=True) remote_client.log_command_output(output, error) percentage = _get_disk_usage_percentage(remote_client) processes1 = remote_client.get_running_processes() output, error = remote_client.execute_command("/etc/init.d/couchbase-server restart", use_channel=True) remote_client.log_command_output(output, error) finally: output, error = remote_client.execute_command_raw("rm -rf full_disk*", use_channel=True) remote_client.log_command_output(output, error) remote_client.disconnect()
def test_rotateInterval(self): intervalSec = self.input.param("intervalSec", None) auditIns = audit(host=self.master) rest = RestConnection(self.master) originalInt = auditIns.getAuditRotateInterval() try: firstEventTime = self.getTimeStampForFile(auditIns) self.log.info ("first time evetn is {0}".format(firstEventTime)) auditIns.setAuditRotateInterval(intervalSec) self.sleep(intervalSec + 20, 'Sleep for log roll over to happen') status, content = rest.validateLogin(self.master.rest_username, self.master.rest_password, True, getContent=True) self.sleep(120) shell = RemoteMachineShellConnection(self.master) try: hostname = shell.execute_command("hostname") archiveFile = hostname[0][0] + '-' + firstEventTime + "-audit.log" self.log.info ("Archive File Name is {0}".format(archiveFile)) result = shell.file_exists(auditIns.pathLogFile, archiveFile) self.assertTrue(result, "Archive Audit.log is not created on time interval") self.log.info ("Validation of archive File created is True, Audit archive File is created {0}".format(archiveFile)) result = shell.file_exists(auditIns.pathLogFile, auditIns.AUDITLOGFILENAME) self.assertTrue(result, "Audit.log is not created when memcached server is killed") finally: shell.disconnect() finally: auditIns.setAuditRotateInterval(originalInt)
def test_rotateIntervalCluster(self): intervalSec = self.input.param("intervalSec", None) nodes_init = self.input.param("nodes_init", 2) auditIns = audit(host=self.master) auditIns.setAuditEnable('true') originalInt = auditIns.getAuditRotateInterval() auditIns.setAuditRotateInterval(intervalSec) firstEventTime = [] try: for i in range(len(self.servers[:nodes_init])): auditTemp = audit(host=self.servers[i]) firstEventTime.append(self.getTimeStampForFile(auditTemp)) self.sleep(intervalSec + 20, 'Sleep for log roll over to happen') for i in range(len(self.servers[:nodes_init])): shell = RemoteMachineShellConnection(self.servers[i]) rest = RestConnection(self.servers[i]) status, content = rest.validateLogin(self.master.rest_username, self.master.rest_password, True, getContent=True) self.sleep(120, "sleeping for log file creation") try: hostname = shell.execute_command("hostname") self.log.info ("print firstEventTime {0}".format(firstEventTime[i])) archiveFile = hostname[0][0] + '-' + firstEventTime[i] + "-audit.log" self.log.info ("Archive File Name is {0}".format(archiveFile)) result = shell.file_exists(auditIns.pathLogFile, archiveFile) self.assertTrue(result, "Archive Audit.log is not created on time interval") self.log.info ("Validation of archive File created is True, Audit archive File is created {0}".format(archiveFile)) result = shell.file_exists(auditIns.pathLogFile, auditIns.AUDITLOGFILENAME) self.assertTrue(result, "Audit.log is not created as per the roll over time specified") finally: shell.disconnect() finally: auditIns.setAuditRotateInterval(originalInt)
def initialize(self, params): start_time = time.time() server = params["server"] remote_client = RemoteMachineShellConnection(params["server"]) replace_127_0_0_1_cmd = "sed -i 's/127.0.0.1/0.0.0.0/g' {0}".format( testconstants.COUCHBASE_SINGLE_DEFAULT_INI_PATH) o, r = remote_client.execute_command(replace_127_0_0_1_cmd) remote_client.log_command_output(o, r) remote_client.stop_couchbase() remote_client.start_couchbase() remote_client.disconnect() couchdb_ok = False while time.time() < (start_time + 60): try: couch_ip = "http://{0}:5984/".format(server.ip) log.info("connecting to couch @ {0}".format(couch_ip)) couch = couchdb.Server(couch_ip) couch.config() # TODO: verify version number and other properties couchdb_ok = True break except Exception as ex: msg = "error happened while creating connection to couchbase single server @ {0} , error : {1}" log.error(msg.format(server.ip, ex)) log.info('sleep for 5 seconds before trying again ...') time.sleep(5) if not couchdb_ok: raise Exception("unable to initialize couchbase single server")
def test_AuditEvent(self): auditIns = audit(host=self.master) ops = self.input.param("ops", None) source = 'internal' user = '******' rest = RestConnection(self.master) #status = rest.setAuditSettings(enabled='true') auditIns.setAuditEnable('true') if (ops in ['enable', 'disable']): if ops == 'disable': #status = rest.setAuditSettings(enabled='false') auditIns.setAuditEnable('false') else: #status = rest.setAuditSettings(enabled='true') auditIns.setAuditEnable('true') if ops == 'disable': shell = RemoteMachineShellConnection(self.master) try: result = shell.file_exists(auditIns.getAuditLogPath(), auditIns.AUDITLOGFILENAME) finally: shell.disconnect() self.assertFalse(result, 'Issue with file getting create in new directory') else: auditIns = audit(host=self.master) expectedResults = {"auditd_enabled":auditIns.getAuditStatus(), "descriptors_path":self.changePathWindows(auditIns.getAuditConfigElement('descriptors_path')), "log_path":self.changePathWindows((auditIns.getAuditLogPath())[:-1]), "source":"internal", "user":"******", "rotate_interval":86400, "version":1, 'hostname':self.getHostName(self.master)} self.checkConfig(self.AUDITCONFIGRELOAD, self.master, expectedResults)
def test_upgrade(self): self._install([self.master]) self.operations([self.master]) for upgrade_version in self.upgrade_versions: self.sleep(self.sleep_time, "Pre-setup of old version is done. Wait for upgrade to {0} version".\ format(upgrade_version)) upgrade_threads = self._async_update(upgrade_version, [self.master]) #wait upgrade statuses for upgrade_thread in upgrade_threads: upgrade_thread.join() success_upgrade = True while not self.queue.empty(): success_upgrade &= self.queue.get() if not success_upgrade: self.fail("Upgrade failed!") self.sleep(self.expire_time) # if not self.is_linux: # self.wait_node_restarted(self.master, wait_time=1200, wait_if_warmup=True, check_service=True) remote = RemoteMachineShellConnection(self.master) for bucket in self.buckets: remote.execute_cbepctl(bucket, "", "set flush_param", "exp_pager_stime", 5) remote.disconnect() self.sleep(30) self.verification([self.master])
def _kill_nodes(self, nodes, servers, bucket_name): self.reboot = self.input.param("reboot", True) if not self.reboot: for node in nodes: _node = { "ip": node.ip, "port": node.port, "username": self.servers[0].rest_username, "password": self.servers[0].rest_password, } node_rest = RestConnection(_node) _mc = MemcachedClientHelper.direct_client(_node, bucket_name) self.log.info("restarted the node %s:%s" % (node.ip, node.port)) pid = _mc.stats()["pid"] command = 'os:cmd("kill -9 {0} ")'.format(pid) self.log.info(command) killed = node_rest.diag_eval(command) self.log.info("killed ?? {0} ".format(killed)) _mc.close() else: for server in servers: shell = RemoteMachineShellConnection(server) command = "reboot" output, error = shell.execute_command(command) shell.log_command_output(output, error) shell.disconnect() time.sleep(self.wait_timeout * 8) shell = RemoteMachineShellConnection(server) command = "/sbin/iptables -F" output, error = shell.execute_command(command) shell.log_command_output(output, error) shell.disconnect()
def test_warmup(self): index_field = self.input.param("index_field", 'name') indexes = [] try: indexes = self._create_multiple_indexes(index_field) num_srv_warm_up = self.input.param("srv_warm_up", 1) if self.input.tuq_client is None: self.fail("For this test external tuq server is requiered. " + \ "Please specify one in conf") self.test_union_all() for server in self.servers[self.nodes_init - num_srv_warm_up:self.nodes_init]: remote = RemoteMachineShellConnection(server) remote.stop_server() remote.start_server() remote.disconnect() #run query, result may not be as expected, but tuq shouldn't fail try: self.test_union_all() except: pass ClusterOperationHelper.wait_for_ns_servers_or_assert(self.servers, self) self.sleep(5) self.test_union_all() finally: self._delete_multiple_indexes(indexes)
def testClusterInit(self): cluster_init_username = self.input.param("cluster_init_username", "Administrator") cluster_init_password = self.input.param("cluster_init_password", "password") cluster_init_port = self.input.param("cluster_init_port", 8091) cluster_init_ramsize = self.input.param("cluster_init_ramsize", 300) command_init = self.input.param("command_init", "cluster-init") param_prefix = self.input.param("param_prefix", "--cluster-init") server = self.servers[-1] remote_client = RemoteMachineShellConnection(server) rest = RestConnection(server) rest.force_eject_node() self.sleep(5) try: cli_command = command_init options = "--cluster-init-username={0} {1}-password={2} {3}-port={4} {5}-ramsize={6}".\ format(cluster_init_username, param_prefix, cluster_init_password, param_prefix, cluster_init_port, param_prefix, cluster_init_ramsize) output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, options=options, cluster_host="localhost", user="******", password="******") self.assertEqual(output[0], "SUCCESS: init localhost") options = "{0}-username={1} {2}-password={3} {4}-port={5}".\ format(param_prefix, cluster_init_username + "1", param_prefix, cluster_init_password + "1", param_prefix, str(cluster_init_port)[:-1] + "9") output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, options=options, cluster_host="localhost", user=cluster_init_username, password=cluster_init_password) # MB-8202 cluster-init/edit doesn't provide status self.assertTrue(output == []) server.rest_username = cluster_init_username + "1" server.rest_password = cluster_init_password + "1" server.port = str(cluster_init_port)[:-1] + "9" cli_command = "server-list" output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, cluster_host="localhost", cluster_port=str(cluster_init_port)[:-1] + "9", user=cluster_init_username + "1", password=cluster_init_password + "1") self.assertTrue("{0} healthy active".format(str(cluster_init_port)[:-1] + "9") in output[0]) server_info = self._get_cluster_info(remote_client, cluster_port=server.port, user=server.rest_username, password=server.rest_password) result = server_info["otpNode"] + " " + server_info["hostname"] + " " + server_info["status"] + " " + server_info["clusterMembership"] self.assertTrue("{0} healthy active".format(str(cluster_init_port)[:-1] + "9") in result) cli_command = command_init options = "{0}-username={1} {2}-password={3} {4}-port={5}".\ format(param_prefix, cluster_init_username, param_prefix, cluster_init_password, param_prefix, cluster_init_port) output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, options=options, cluster_host="localhost", cluster_port=str(cluster_init_port)[:-1] + "9", user=(cluster_init_username + "1"), password=cluster_init_password + "1") # MB-8202 cluster-init/edit doesn't provide status self.assertTrue(output == []) server.rest_username = cluster_init_username server.rest_password = cluster_init_password server.port = cluster_init_port remote_client = RemoteMachineShellConnection(server) cli_command = "server-list" output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, cluster_host="localhost", user=cluster_init_username, password=cluster_init_password) self.assertTrue("{0} healthy active".format(str(cluster_init_port)) in output[0]) server_info = self._get_cluster_info(remote_client, cluster_port=server.port, user=server.rest_username, password=server.rest_password) result = server_info["otpNode"] + " " + server_info["hostname"] + " " + server_info["status"] + " " + server_info["clusterMembership"] self.assertTrue("{0} healthy active".format(str(cluster_init_port)) in result) remote_client.disconnect() finally: rest = RestConnection(server) rest.force_eject_node() self.sleep(5) rest.init_cluster()
def replication_while_rebooting_a_non_master_destination_node(self): self.set_xdcr_param("xdcrFailureRestartInterval", 1) self._load_all_buckets(self.src_master, self.gen_create, "create", 0) self._async_modify_data() self.sleep(self._timeout) i = len(self.dest_nodes) - 1 shell = RemoteMachineShellConnection(self.dest_nodes[i]) type = shell.extract_remote_info().type.lower() if type == 'windows': o, r = shell.execute_command("shutdown -r -f -t 0") elif type == 'linux': o, r = shell.execute_command("reboot") shell.log_command_output(o, r) shell.disconnect() self.sleep(60, "after rebooting node") num = 0 while num < 10: try: shell = RemoteMachineShellConnection(self.dest_nodes[i]) except BaseException, e: self.log.warn("node {0} is unreachable".format(self.dest_nodes[i].ip)) self.sleep(60, "still can't connect to node") num += 1 else: break
def _stop_moxi(self): self.log.info("kill moxi server at %s " % self.moxi_server.ip) command = "kill -9 $(ps aux | grep -v grep | grep {0} | awk '{{print $2}}')".format(self.moxi_port) shell = RemoteMachineShellConnection(self.moxi_server) output, error = shell.execute_command_raw(command) shell.log_command_output(output, error) shell.disconnect()
def user_manage(self, delete, list, set, ro_username, ro_password): options = self._get_default_options() if delete: options += " --delete " if list: options += " --list " if set: options += " --set " if ro_username is not None: options += " --ro-username " + str(ro_username) if ro_password: options += " --ro-password " + str(ro_password) remote_client = RemoteMachineShellConnection(self.server) stdout, stderr = remote_client.couchbase_cli("user-manage", self.hostname, options) remote_client.disconnect() if delete: return stdout, stderr, self._was_success(stdout, "Local read-only" "user deleted") elif set: return stdout, stderr, self._was_success(stdout, "Local read-only" "user deleted") else: return stdout, stderr, self._no_error_in_output(stdout)
def test_folderMisMatchCluster(self): auditIns = audit(host=self.master) orginalPath = auditIns.getAuditLogPath() newPath = originalPath + 'testFolderMisMatch' shell = RemoteMachineShellConnection(self.servers[0]) try: shell.create_directory(newPath) command = 'chown couchbase:couchbase ' + newPath shell.execute_command(command) finally: shell.disconnect() auditIns.setsetAuditLogPath(newPath) for server in self.servers: rest = RestConnection(sever) #Create an Event for Bucket Creation expectedResults = {'name':'TestBucket ' + server.ip, 'ram_quota':536870912, 'num_replicas':1, 'replica_index':False, 'eviction_policy':'value_only', 'type':'membase', \ 'auth_type':'sasl', "autocompaction":'false', "purge_interval":"undefined", \ "flush_enabled":False, "num_threads":3, "source":source, \ "user":user, "ip":self.ipAddress, "port":57457, 'sessionid':'' } rest.create_bucket(expectedResults['name'], expectedResults['ram_quota'] / 1048576, expectedResults['auth_type'], 'password', expectedResults['num_replicas'], \ '11211', 'membase', 0, expectedResults['num_threads'], expectedResults['flush_enabled'], 'valueOnly') #Check on Events try: self.checkConfig(self.eventID, self.servers[0], expectedResults) except: self.log.info ("Issue reading the file at Node {0}".format(server.ip))
def _setting_cluster(self, cmd, data_ramsize, index_ramsize, fts_ramsize, cluster_name, cluster_username, cluster_password, cluster_port): options = self._get_default_options() if cluster_username is not None: options += " --cluster-username " + str(cluster_username) if cluster_password is not None: options += " --cluster-password " + str(cluster_password) if data_ramsize: options += " --cluster-ramsize " + str(data_ramsize) if index_ramsize: options += " --cluster-index-ramsize " + str(index_ramsize) if fts_ramsize: options += " --cluster-fts-ramsize " + str(fts_ramsize) if cluster_name: options += " --cluster-name " + str(cluster_name) if cluster_port: options += " --cluster-port " + str(cluster_port) remote_client = RemoteMachineShellConnection(self.server) stdout, stderr = remote_client.couchbase_cli(cmd, self.hostname, options) remote_client.disconnect() return stdout, stderr, self._was_success(stdout, "Cluster settings modified")
def setting_compaction(self, db_frag_perc, db_frag_size, view_frag_perc, view_frag_size, from_period, to_period, abort_outside, parallel_compact, purgeint): options = self._get_default_options() if db_frag_perc is not None: options += " --compaction-db-percentage " + str(db_frag_perc) if db_frag_size is not None: options += " --compaction-db-size " + str(db_frag_size) if view_frag_perc is not None: options += " --compaction-view-percentage " + str(view_frag_perc) if view_frag_size is not None: options += " --compaction-view-size " + str(view_frag_size) if from_period is not None: options += " --compaction-period-from " + str(from_period) if to_period is not None: options += " --compaction-period-to " + str(to_period) if abort_outside is not None: options += " --enable-compaction-abort " + str(abort_outside) if parallel_compact is not None: options += " --enable-compaction-parallel " + str(parallel_compact) if purgeint is not None: options += " --metadata-purge-interval " + str(purgeint) remote_client = RemoteMachineShellConnection(self.server) stdout, stderr = remote_client.couchbase_cli("setting-compaction", self.hostname, options) remote_client.disconnect() return stdout, stderr, self._was_success(stdout, "Compaction " "settings modified")
def cluster_init(self, data_ramsize, index_ramsize, fts_ramsize, services, index_storage_mode, cluster_name, cluster_username, cluster_password, cluster_port): options = "" if cluster_username: options += " --cluster-username " + str(cluster_username) if cluster_password: options += " --cluster-password " + str(cluster_password) if data_ramsize: options += " --cluster-ramsize " + str(data_ramsize) if index_ramsize: options += " --cluster-index-ramsize " + str(index_ramsize) if fts_ramsize: options += " --cluster-fts-ramsize " + str(fts_ramsize) if cluster_name: options += " --cluster-name " + str(cluster_name) if index_storage_mode: options += " --index-storage-setting " + str(index_storage_mode) if cluster_port: options += " --cluster-port " + str(cluster_port) if services: options += " --services " + str(services) remote_client = RemoteMachineShellConnection(self.server) stdout, stderr = remote_client.couchbase_cli("cluster-init", self.hostname, options) remote_client.disconnect() return stdout, stderr, self._was_success(stdout, "Cluster initialized")
def testClusterInitNegative(self): cluster_init_username = self.input.param("cluster_init_username", None) cluster_init_password = self.input.param("cluster_init_password", None) cluster_init_port = self.input.param("cluster_init_port", None) cluster_init_ramsize = self.input.param("cluster_init_ramsize", None) command_init = self.input.param("command_init", "cluster-init") server = self.servers[-1] remote_client = RemoteMachineShellConnection(server) rest = RestConnection(server) rest.force_eject_node() self.sleep(5) try: cli_command = command_init options = "" if cluster_init_username is not None: options += "--cluster-init-username={0} ".format(cluster_init_username) if cluster_init_password is not None: options += "--cluster-init-password={0} ".format(cluster_init_password) if cluster_init_port is not None: options += "--cluster-init-port={0} ".format(cluster_init_port) if cluster_init_ramsize is None: options += "--cluster-init-ramsize={0} ".format(cluster_init_ramsize) output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, options=options, cluster_host="localhost", user=None, password=None) self.assertEqual(output[0], 'ERROR: unable to init localhost (400) Bad Request') self.assertTrue(output[1] == "[u'Username and password are required.']" or output[1] == "[u'The password must be at least six characters.']") remote_client.disconnect() finally: rest = RestConnection(server) rest.force_eject_node() self.sleep(5) rest.init_cluster()
def rebalance_stop(self): options = self._get_default_options() remote_client = RemoteMachineShellConnection(self.server) stdout, stderr = remote_client.couchbase_cli("rebalance-stop", self.hostname, options) remote_client.disconnect() return stdout, stderr, self._was_success(stdout, "Rebalance stopped")
def mutate_and_check_error404(self, n=1): # get vb0 active source node active_src_node = self.get_active_vb0_node(self.src_master) shell = RemoteMachineShellConnection(active_src_node) os_type = shell.extract_remote_info().distribution_type if os_type.lower() == 'windows': trace_log = "C:/Program Files/Couchbase/Server/var/lib/couchbase/logs/xdcr_trace.log" else: trace_log = "/opt/couchbase/var/lib/couchbase/logs/xdcr_trace.*" num_404_errors_before_load, error = shell.execute_command("grep \"error,404\" {} | wc -l" .format(trace_log)) num_get_remote_bkt_failed_before_load, error = shell.execute_command("grep \"get_remote_bucket_failed\" \"{}\" | wc -l" .format(trace_log)) self.log.info("404 errors: {}, get_remote_bucket_failed errors : {}". format(num_404_errors_before_load, num_get_remote_bkt_failed_before_load)) self.sleep(60) self.log.info("################ New mutation:{} ##################".format(self.key_counter+1)) self.load_one_mutation_into_source_vb0(active_src_node) self.sleep(5) num_404_errors_after_load, error = shell.execute_command("grep \"error,404\" {} | wc -l" .format(trace_log)) num_get_remote_bkt_failed_after_load, error = shell.execute_command("grep \"get_remote_bucket_failed\" \"{}\" | wc -l" .format(trace_log)) self.log.info("404 errors: {}, get_remote_bucket_failed errors : {}". format(num_404_errors_after_load, num_get_remote_bkt_failed_after_load)) shell.disconnect() if (int(num_404_errors_after_load[0]) > int(num_404_errors_before_load[0])) or \ (int(num_get_remote_bkt_failed_after_load[0]) > int(num_get_remote_bkt_failed_before_load[0])): self.log.info("Checkpointing error-404 verified after dest failover/rebalance out") return True else: self.log.info("404 errors on source node before last load : {}, after last node: {}". format(int(num_404_errors_after_load[0]), int(num_404_errors_before_load[0]))) self.log.error("Checkpoint 404 error NOT recorded at source following dest failover or rebalance!")
def rebalance_in_with_DB_time_compaction(self): remote_client = RemoteMachineShellConnection(self.master) rest = RestConnection(self.master) currTime = datetime.datetime.now() fromTime = currTime + datetime.timedelta(hours=1) toTime = currTime + datetime.timedelta(hours=24) self.set_auto_compaction(rest, dbFragmentThresholdPercentage=self.autocompaction_value, allowedTimePeriodFromHour=fromTime.hour, allowedTimePeriodFromMin=fromTime.minute, allowedTimePeriodToHour=toTime.hour, allowedTimePeriodToMin=toTime.minute, allowedTimePeriodAbort="false") self._load_all_buckets(self.master, self.gen_load, "create", 0, 1) self._monitor_DB_fragmentation() for i in xrange(10): active_tasks = self.cluster.async_monitor_active_task(self.master, "bucket_compaction", "bucket", wait_task=False) for active_task in active_tasks: result = active_task.result() self.assertTrue(result) self.sleep(2) currTime = datetime.datetime.now() #Need to make it configurable newTime = currTime + datetime.timedelta(minutes=5) self.set_auto_compaction(rest, dbFragmentThresholdPercentage=self.autocompaction_value, allowedTimePeriodFromHour=currTime.hour, allowedTimePeriodFromMin=currTime.minute, allowedTimePeriodToHour=newTime.hour, allowedTimePeriodToMin=newTime.minute, allowedTimePeriodAbort="false") servs_in = self.servers[self.nodes_init:self.nodes_in + 1] rebalance = self.cluster.async_rebalance([self.master], servs_in, []) compact_run = remote_client.wait_till_compaction_end(rest, self.default_bucket_name, timeout_in_seconds=(self.wait_timeout * 5)) rebalance.result() if compact_run: self.log.info("auto compaction run successfully") else: self.fail("auto compaction does not run") remote_client.disconnect()
def verify_for_recovery_type(self, chosen = [], serverMap = {}, buckets = [], recoveryTypeMap = {}, fileMap = {}, deltaRecoveryBuckets = []): """ Verify recovery type is delta or full """ summary = "" logic = True for server in self.chosen: shell = RemoteMachineShellConnection(serverMap[server.ip]) os_type = shell.extract_remote_info() if os_type.type.lower() == 'windows': return for bucket in buckets: path = fileMap[server.ip][bucket.name] exists = shell.file_exists(path,"check.txt") if deltaRecoveryBuckets != None: if recoveryTypeMap[server.ip] == "delta" and (bucket.name in deltaRecoveryBuckets) and not exists: logic = False summary += "\n Failed Condition :: node {0}, bucket {1} :: Expected Delta, Actual Full".format(server.ip,bucket.name) elif recoveryTypeMap[server.ip] == "delta" and (bucket.name not in deltaRecoveryBuckets) and exists: summary += "\n Failed Condition :: node {0}, bucket {1} :: Expected Full, Actual Delta".format(server.ip,bucket.name) logic = False else: if recoveryTypeMap[server.ip] == "delta" and not exists: logic = False summary += "\n Failed Condition :: node {0}, bucket {1} :: Expected Delta, Actual Full".format(server.ip,bucket.name) elif recoveryTypeMap[server.ip] == "full" and exists: logic = False summary += "\n Failed Condition :: node {0}, bucket {1} :: Expected Full, Actual Delta".format(server.ip,bucket.name) shell.disconnect() self.assertTrue(logic, summary)
def rebalance_in_out_with_auto_DB_compaction(self): remote_client = RemoteMachineShellConnection(self.master) rest = RestConnection(self.master) self.assertTrue(self.num_servers > self.nodes_in + self.nodes_out, "ERROR: Not enough nodes to do rebalance in and out") servs_init = self.servers[:self.nodes_init] servs_in = [self.servers[i + self.nodes_init] for i in range(self.nodes_in)] servs_out = [self.servers[self.nodes_init - i - 1] for i in range(self.nodes_out)] result_nodes = set(servs_init + servs_in) - set(servs_out) self.set_auto_compaction(rest, dbFragmentThresholdPercentage=self.autocompaction_value) self._load_all_buckets(self.master, self.gen_load, "create", 0, 1) rebalance = self.cluster.async_rebalance(servs_init, servs_in, servs_out) while rebalance.state != "FINISHED": self._monitor_DB_fragmentation() compact_run = remote_client.wait_till_compaction_end(rest, self.default_bucket_name, timeout_in_seconds=(self.wait_timeout * 5)) rebalance.result() monitor_fragm = self.cluster.async_monitor_db_fragmentation(self.master, 0, self.default_bucket_name) result = monitor_fragm.result() if compact_run: self.log.info("auto compaction run successfully") elif result: self.log.info("Compaction is already completed") else: self.fail("auto compaction does not run") self.verify_cluster_stats(result_nodes) remote_client.disconnect()
def offline_cluster_upgrade_with_reinstall(self): self._install(self.servers[:self.nodes_init]) self.operations(self.servers[:self.nodes_init]) if self.ddocs_num: self.create_ddocs_and_views() if self.during_ops: for opn in self.during_ops: getattr(self, opn)() num_nodes_reinstall = self.input.param('num_nodes_reinstall', 1) stoped_nodes = self.servers[self.nodes_init - (self.nodes_init - num_nodes_reinstall):self.nodes_init] nodes_reinstall = self.servers[:num_nodes_reinstall] for upgrade_version in self.upgrade_versions: self.sleep(self.sleep_time, "Pre-setup of old version is done. Wait for upgrade to {0} version".\ format(upgrade_version)) for server in stoped_nodes: remote = RemoteMachineShellConnection(server) remote.stop_server() remote.disconnect() self.sleep(self.sleep_time) upgrade_threads = self._async_update(upgrade_version, stoped_nodes) self.force_reinstall(nodes_reinstall) for upgrade_thread in upgrade_threads: upgrade_thread.join() success_upgrade = True while not self.queue.empty(): success_upgrade &= self.queue.get() if not success_upgrade: self.fail("Upgrade failed!") self.dcp_rebalance_in_offline_upgrade_from_version2_to_version3() self.verification(self.servers[:self.nodes_init])
def offline_cluster_upgrade(self): self._install(self.servers[:self.nodes_init]) self.operations(self.servers[:self.nodes_init]) seqno_expected = 1 if self.ddocs_num: self.create_ddocs_and_views() if self.input.param('run_index', False): self.verify_all_queries() if not self.initial_version.startswith("1.") and self.input.param('check_seqno', True): self.check_seqno(seqno_expected) if self.during_ops: for opn in self.during_ops: if opn != 'add_back_failover': getattr(self, opn)() num_stoped_nodes = self.input.param('num_stoped_nodes', self.nodes_init) upgrade_nodes = self.servers[self.nodes_init - num_stoped_nodes :self.nodes_init] for upgrade_version in self.upgrade_versions: self.sleep(self.sleep_time, "Pre-setup of old version is done. Wait for upgrade to {0} version".\ format(upgrade_version)) for server in upgrade_nodes: remote = RemoteMachineShellConnection(server) remote.stop_server() self.sleep(self.sleep_time) if self.wait_expire: self.sleep(self.expire_time) if self.input.param('remove_manifest_files', False): for file in ['manifest.txt', 'manifest.xml', 'VERSION.txt,']: output, error = remote.execute_command("rm -rf /opt/couchbase/{0}".format(file)) remote.log_command_output(output, error) if self.input.param('remove_config_files', False): for file in ['config', 'couchbase-server.node', 'ip', 'couchbase-server.cookie']: output, error = remote.execute_command("rm -rf /opt/couchbase/var/lib/couchbase/{0}".format(file)) remote.log_command_output(output, error) self.buckets = [] remote.disconnect() upgrade_threads = self._async_update(upgrade_version, upgrade_nodes) #wait upgrade statuses for upgrade_thread in upgrade_threads: upgrade_thread.join() success_upgrade = True while not self.queue.empty(): success_upgrade &= self.queue.get() if not success_upgrade: self.fail("Upgrade failed. See logs above!") self.sleep(self.expire_time) if self.during_ops: if "add_back_failover" in self.during_ops: getattr(self, 'add_back_failover')() self.cluster.rebalance(self.servers[:self.nodes_init], [], []) elif "failover" in self.during_ops: self.cluster.rebalance(self.servers[:self.nodes_init], [], []) rem = [server for server in self.servers[:self.nodes_init] if self.failover_node.ip == server.ip and str(self.failover_node.port) == server.port] self.dcp_rebalance_in_offline_upgrade_from_version2_to_version3() self.verification(list(set(self.servers[:self.nodes_init]) - set(rem))) return self.dcp_rebalance_in_offline_upgrade_from_version2_to_version3() self.verification(self.servers[:self.nodes_init]) if self.input.param('check_seqno', True): self.check_seqno(seqno_expected)
def setting_index(self, max_rollbacks, stable_snap_interval, mem_snap_interval, storage_mode, threads, log_level): options = self._get_default_options() if max_rollbacks: options += " --index-max-rollback-points " + str(max_rollbacks) if stable_snap_interval: options += " --index-stable-snapshot-interval " + str( stable_snap_interval) if mem_snap_interval: options += " --index-memory-snapshot-interval " + str( mem_snap_interval) if storage_mode: options += " --index-storage-setting " + str(storage_mode) if threads: options += " --index-threads " + str(threads) if log_level: options += " --index-log-level " + str(log_level) remote_client = RemoteMachineShellConnection(self.server) stdout, stderr = remote_client.couchbase_cli("setting-index", self.hostname, options) remote_client.disconnect() return stdout, stderr, self._was_success(stdout, "Indexer settings modified")
def bucket_create(self, name, password, bucket_type, quota, eviction_policy, replica_count, enable_replica_indexes, priority, enable_flush, wait): options = self._get_default_options() if name is not None: options += " --bucket " + name if password is not None: options += " --bucket-password " + password if bucket_type is not None: options += " --bucket-type " + bucket_type if quota is not None: options += " --bucket-ramsize " + str(quota) if eviction_policy is not None: options += " --bucket-eviction-policy " + eviction_policy if replica_count is not None: options += " --bucket-replica " + str(replica_count) if enable_replica_indexes is not None: options += " --enable-index-replica " + str(enable_replica_indexes) if priority is not None: options += " --bucket-priority " + priority if enable_flush is not None: options += " --enable-flush " + str(enable_flush) if wait: options += " --wait" remote_client = RemoteMachineShellConnection(self.server) stdout, stderr = remote_client.couchbase_cli("bucket-create", self.hostname, options) remote_client.disconnect() return stdout, stderr, self._was_success(stdout, "Bucket created")
def run_ops(self): tasks = [] if not self.ops: return tasks if self.ops == 'rebalance_in': tasks.append(self.cluster.async_rebalance(self.servers[:self.nodes_init], self.servers[self.nodes_init:self.nodes_init + self.nodes_in], [])) self.nodes_init += self.nodes_in elif self.ops == 'rebalance_out': tasks.append(self.cluster.async_rebalance(self.servers[:self.nodes_init], [], self.servers[(self.nodes_init - self.nodes_out):self.nodes_init])) self.nodes_init -= self.nodes_out elif self.ops == 'failover': tasks.append(self.cluster.failover(self.servers[:self.nodes_init], self.servers[(self.nodes_init - self.nodes_out):self.nodes_init])) self.sleep(20) self.nodes_init -= self.nodes_out if self.ops == 'create_views': views_num = 10 views = self.make_default_views(self.view_name, views_num, different_map=True) tasks.extend(self.async_create_views(self.master, self.ddoc_name, views)) if self.ops == 'restart': servers_to_choose = [serv for serv in self.servers if self.master.ip != serv.ip] self.assertTrue(servers_to_choose, "There is only one node in cluster") shell = RemoteMachineShellConnection(servers_to_choose[0]) try: shell.stop_couchbase() shell.start_couchbase() finally: shell.disconnect() self.sleep(5, "Server %s is starting..." % servers_to_choose[0].ip) return tasks
def test_large_file_version(self): rest = RestConnection(self.master) remote_client = RemoteMachineShellConnection(self.master) remote_client.extract_remote_info() self._load_all_buckets(self.master, self.gen_load, "create", 0, 1) self.disable_compaction() self._monitor_DB_fragmentation() # rename here remote_client.stop_couchbase() time.sleep(5) remote_client.execute_command("cd /opt/couchbase/var/lib/couchbase/data/default;rename .1 .65535 *.1") remote_client.execute_command("cd /opt/couchbase/var/lib/couchbase/data/default;rename .2 .65535 *.2") remote_client.start_couchbase() for i in range(5): self.log.info("starting a compaction iteration") compaction_task = self.cluster.async_compact_bucket(self.master, self.default_bucket_name) compact_run = remote_client.wait_till_compaction_end(rest, self.default_bucket_name, timeout_in_seconds=self.wait_timeout) res = compaction_task.result(self.wait_timeout) if compact_run: self.log.info("auto compaction run successfully") else: self.fail("auto compaction does not run") remote_client.disconnect()
def create_user(self): userCreateCmmd = 'dn: cn=' + self.user_name + "," + self.LDAP_DN + "\n" \ "cn:" + self.user_name + "\n" \ "sn: " + self.user_name + "\n" \ 'objectClass: ' + self.LDAP_OBJECT_CLASS + "\n" \ "userPassword :"******"\n" \ "uid: " + self.user_name + "\n" fileName = 'name.ldif' # Execute ldapadd command to add users to the system shell = RemoteMachineShellConnection(self.host) o = '' r = '' try: shell.write_remote_file("/tmp", fileName, userCreateCmmd) command = "ldapadd -h " + self.LDAP_HOST + " -p " + self.LDAP_PORT + " -f /tmp/" + fileName + " -D " + \ self.LDAP_ADMIN_USER + " -w " + self.LDAP_ADMIN_PASS o, r = shell.execute_command(command) shell.log_command_output(o, r) command = "rm -rf /tmp/*.ldif" o, r = shell.execute_command(command) shell.log_command_output(o, r) finally: shell.disconnect() return o
def testSGServiceInstallHelp(self): shell = RemoteMachineShellConnection(self.master) self.kill_processes_gateway(shell) self.uninstall_gateway(shell) self.install_gateway(shell) output, error = self.run_sync_gateway_service_install(shell, "-h") self.assertEqual(error, []) self.assertEqual(output, help) output, error = self.run_sync_gateway_service_install(shell) self.assertEqual( error[0], "The sync_gateway runtime user account does not exist \"sync_gateway\"." ) self.assertEqual(output, []) output, error = self.run_sync_gateway_service_install( shell, "bla-bla-bla") temp_help = ["ERROR: unknown parameter \"bla-bla-bla\""] temp_help.extend(help) self.assertEqual(error, []) self.assertEqual(output, temp_help) shell.disconnect()
def test_software_version(self): """ This test requires to pass 3 params to run: software_name software_version check_in_file """ self.software_name = self.input.param("software_name", None) self.software_version = self.input.param("software_version", None) self.check_in_file = self.input.param("check_in_file", "manifest.xml") if self.software_name is None or self.software_version is None: self.fail("This test needs to pass param 'software_name'\ and software_version to run") go_software = ["gocb", "gocbcore"] go_sw_in_version = ["5.1.2", "5.5.1"] if self.software_name in go_software and \ (self.cb_version[:5] in go_sw_in_version or 6.0 <= float(self.cb_version[:3])): shell = RemoteMachineShellConnection(self.master) output, error = shell.execute_command( 'cat {0}/{1} | grep \'"{2}"\' '.format(self.base_cb_path, self.check_in_file, self.software_name)) shell.disconnect() found_version = False if output: for ele in output: if "gocb" in ele and self.software_version in ele: found_version = True self.log.info("software info: {0}".format(ele)) break if not found_version: self.fail("version of {0} does not match as in: {0}"\ .format(self.software_name, output)) else: self.log.info( "software name/version are not in running cb version")
def test_offline_upgrade(self): upgrade_nodes = self.servers[:self.nodes_init] if self.disable_plasma_upgrade: self._install(self.nodes_in_list, version=self.upgrade_to) rebalance = self.cluster.async_rebalance( self.servers[:self.nodes_init], [self.nodes_in_list[0]], [], services=["index"]) rebalance.result() self.sleep(100) self.disable_upgrade_to_plasma(self.nodes_in_list[0]) for server in upgrade_nodes: remote = RemoteMachineShellConnection(server) remote.stop_server() remote.disconnect() upgrade_threads = self._async_update(self.upgrade_to, [server]) for upgrade_thread in upgrade_threads: upgrade_thread.join() self.upgrade_servers.append(server) self.sleep(100) msg = "Cluster is not healthy after upgrade" self.assertTrue(self.wait_until_cluster_is_healthy(), msg) log.info("Cluster is healthy") self.assertTrue(self.wait_until_indexes_online(), "Some indexes are not online") log.info("All indexes are online") self.add_built_in_server_user() self.sleep(20) if self.initial_version.split("-")[0] in UPGRADE_VERS: self.multi_drop_index() self.sleep(100) self._create_indexes() self.sleep(100) self._query_index("post_upgrade") self._verify_post_upgrade_results() self._update_int64_dataset() self._query_for_long_num()
def get_views_definition_from_backup_file(self, server, backup_dir, buckets): """ Extract key value from database file shard_0.fdb Return: key, kv store name, status and value /tmp/entbackup/backup/20*/default-*/views.json """ conn = RemoteMachineShellConnection(server) result = False backup_data = {} for bucket in buckets: backup_data[bucket.name] = {} output, _ = conn.execute_command("ls %s/backup/20*/%s* " % (backup_dir, bucket.name)) if "views.json" in output: cmd = "cat %s/backup/20*/%s*/views.json" \ % (backup_dir, bucket.name) views_output, _ = conn.execute_command(cmd) views_output = [x.strip(' ') for x in views_output] if views_output: views_output = " ".join(views_output) result = views_output conn.disconnect() return result
def chmod(self, server, path, mod="000"): ''' # (Base-10) Binary Sum (in binary) Sum (in decimal) rwx Permission 7 111 = 100 + 10 + 1 = 4(r) + 2(w) + 1(x) rwx read, write and execute 6 110 = 100 + 10 = 4(r) + 2(w) rw- read and write 5 101 = 100 + 1 = 4(r) + 1(x) r-x read and execute 4 100 = 100 = 4(r) r-- read only 3 011 = 10 + 1 = 2(w) + 1(x) -wx write and execute 2 010 = 10 = 2(w) -w- write only 1 001 = 1 = 1(x) --x execute only 0 000 = 0 = 0 --- none ''' self.stop_chmod = False while self.stop_chmod is False: shell = RemoteMachineShellConnection(server) self.log.debug("{}: changing mod to {} for {}".format( server.ip, mod, path)) shell.execute_command("chmod {} {}".format(mod, path)) self.sleep(5) self.log.debug("{}: changing mod to {} for {}".format( server.ip, "777", path)) shell.execute_command("chmod {} {}".format("777", path)) self.sleep(5) shell.disconnect()
def change_port(self, new_port="9090", current_port='8091'): nodes = RestConnection(self.cluster.master).node_statuses() remote_client = RemoteMachineShellConnection(self.cluster.master) options = "--cluster-port=%s" % new_port cli_command = "cluster-edit" output, error = remote_client.execute_couchbase_cli( cli_command=cli_command, options=options, cluster_host="localhost:%s" % current_port, user=self.cluster.master.rest_username, password=self.cluster.master.rest_password) self.log.info(output) remote_client.disconnect() # MB-10136 & MB-9991 if error: raise Exception("Port didn't change! %s" % error) self.port = new_port self.log.info("New port '%s' on nodes: %s" % (new_port, [node.ip for node in nodes])) for node in nodes: for server in self.cluster.servers: if server.ip == node.ip and int(server.port) == int(node.port): server.port = new_port break
def set_metadata_purge_interval(self, value, buckets=[], node=None): self.log.info( "Changing the bucket properties by changing {0} to {1}".format( "purge_interval", value)) if not buckets: buckets = self.buckets if node is None: node = self.cluster.master rest = RestConnection(node) shell = RemoteMachineShellConnection(node) shell.enable_diag_eval_on_non_local_hosts() shell.disconnect() for bucket in buckets: cmd = '{ok, BC} = ns_bucket:get_bucket(' \ '"%s"), BC2 = lists:keyreplace(purge_interval, ' \ '1, BC, {purge_interval, %f})' \ ', ns_bucket:set_bucket_config("%s", BC2).' \ % (bucket.name, value, bucket.name) rest.diag_eval(cmd) # Restart Memcached in all cluster nodes to reflect the settings for server in self.cluster_util.get_kv_nodes(self.cluster, master=node): shell = RemoteMachineShellConnection(server) shell.restart_couchbase() shell.disconnect() # Check bucket-warm_up after Couchbase restart retry_count = 10 buckets_warmed_up = self.bucket_util.is_warmup_complete( self.cluster, buckets, retry_count) if not buckets_warmed_up: self.log.critical("Few bucket(s) not warmed up " "within expected time")
def test_prompt_enter_correct_password(self): try: self.secretmgmt_base_obj.set_password(self.master, self.password) shell = RemoteMachineShellConnection(self.master) shell.execute_command("/opt/couchbase/etc/couchbase_init.d stop") shell.disconnect() self.secretmgmt_base_obj.start_server_prompt_diff_window( self.master) self.sleep(10) # self.secretmgmt_base_obj.incorrect_password(self.master, cmd="/opt/couchbase/bin/cbmaster_password", # retries_number=1, input_correct_pass=True, correct_pass='******') cmd = "/opt/couchbase/bin/couchbase-cli master-password -c localhost:8091 -u Administrator -p password --send-password" temp_result = self.secretmgmt_base_obj.correct_password_on_prompt( self.master, self.password, cmd=cmd) self.assertTrue( temp_result, "Issue with passing in correct password on prompt") finally: for server in self.servers: shell = RemoteMachineShellConnection(server) if (RemoteMachineHelper(shell).is_process_running('memcached') is None): shell.set_environment_variable("CB_MASTER_PASSWORD", self.password)
class Examine: def __init__(self, server): self.remote_connection = RemoteMachineShellConnection(server) def examine(self, examine_arguments): """ Returns an ExamineResults given an ExamineArguments object """ if not examine_arguments.json: raise ValueError( "Currently the non-JSON data output from the examine sub-command is not supported for testing." ) output, error, exit_code = self.remote_connection.execute_command( examine_arguments.to_command(), get_exit_code=True) if exit_code != 0 or not output: return None, error return ExamineResult.from_output(output[0]) def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): self.remote_connection.disconnect()
def run(self): remote_client = RemoteMachineShellConnection(self.server) now = datetime.now() day = now.day month = now.month year = now.year hour = now.timetuple().tm_hour min = now.timetuple().tm_min file_name = "%s-%s%s%s-%s%s-diag.zip" % (self.server.ip, month, day, year, hour, min) print "Collecting logs from %s\n" % (self.server.ip) output, error = remote_client.execute_cbcollect_info(file_name) print "\n".join(output) print "\n".join(error) user_path = "/home/" if self.server.ssh_username == "root": user_path = "/" if not remote_client.file_exists("%s%s" % (user_path, self.server.ssh_username), file_name): raise Exception("%s doesn't exists on server" % (file_name)) if remote_client.get_file("%s%s" % (user_path, self.server.ssh_username), file_name, "%s/%s" % (self.path, file_name)): print "Downloading zipped logs from %s" % (self.server.ip) else: raise Exception("Fail to download zipped logs from %s" % (self.server.ip)) remote_client.disconnect()
def configure_bucket_backups(self): self.cluster_util.update_cluster_nodes_service_list(self.cluster) backup_node = self.cluster.backup_nodes[0] backup_helper = BackupHelper(backup_node) self.log.info("Creating permissions for backup folder") backup_configs = self.cluster_config["cb_cluster"]["backup"] shell = RemoteMachineShellConnection(backup_node) for backup_config in backup_configs: plan_params = dict() repo_params = dict() if "plan" in backup_config: plan_params["plan"] = backup_config["plan"] repo_params["plan"] = backup_config["plan"] if "description" in backup_config: plan_params["description"] = backup_config["description"] if "archive_path" in backup_config: repo_params["archive"] = backup_config["archive_path"] shell.execute_command("mkdir -p {0} ; chmod 777 {0}" .format(backup_config["archive_path"])) if "bucket" in backup_config: repo_params["bucket"] = backup_config["bucket"] if plan_params["plan"] not in ["_hourly_backups", "_daily_backups"]: self.log.info("Updating custom plan %s" % plan_params["plan"]) status = backup_helper.create_edit_plan("create", plan_params) if status is False: self.fail("Backup %s create failed" % backup_config) # Create repo status = backup_helper.create_repo(backup_config["repo_id"], repo_params) if status is False: self.fail("Create repo failed for %s" % backup_config) shell.disconnect()
def get_checkpoint_call_history(self, node): shell = RemoteMachineShellConnection(node) os_type = shell.extract_remote_info().distribution_type if os_type.lower() == 'windows': couchdb_log = "C:/Program Files/Couchbase/Server/var/lib/couchbase/logs/couchdb.log" else: couchdb_log = "/opt/couchbase/var/lib/couchbase/logs/couchdb.log" total_chkpt_calls, error = shell.execute_command( "grep \"POST /_commit_for_checkpoint\" \"{0}\" | wc -l".format( couchdb_log)) total_successful_chkpts, error = shell.execute_command( "grep \"POST /_commit_for_checkpoint 200\" \"{0}\" | wc -l".format( couchdb_log)) self.log.info(int(total_successful_chkpts[0])) if self.num_successful_chkpts_so_far != 0: checkpoint_number = int(total_successful_chkpts[0] ) - self.num_successful_chkpts_beginning self.log.info("Checkpoint on this node (this run): {0}".format( checkpoint_number)) shell.disconnect() total_commit_failures = int(total_chkpt_calls[0]) - int( total_successful_chkpts[0]) return int(total_chkpt_calls[0]), int( total_successful_chkpts[0]), total_commit_failures
def checkTLS1_1_blocking(self): self.get_the_testssl_script(self.TEST_SSL_FILENAME) command = "ns_config:set(ssl_minimum_protocol, 'tlsv1.2')" self.log.info("posting: %s" % command) rest = RestConnection(self.master) res = rest.diag_eval(command) # do the initial check self.check_all_servers(rest) # restart the server try: for server in self.servers: shell = RemoteMachineShellConnection(server) shell.stop_couchbase() time.sleep(10) # Avoid using sleep like this on further calls shell.start_couchbase() shell.disconnect() except Exception as e: self.log.error(traceback.format_exc()) # and check again time.sleep(30) self.check_all_servers(rest)
def retrieve_request_status_using_handle(self, server, handle): """ Retrieves status of a request from /analytics/status endpoint """ shell = RemoteMachineShellConnection(server) output, error = shell.execute_command("""curl -v {0}""".format(handle)) response = "" for line in output: response = response + line if response: response = json.loads(response) shell.disconnect() status = "" handle = "" if 'status' in response: status = response['status'] if 'handle' in response: handle = response['handle'] self.log.info("status=%s, handle=%s" % (status, handle)) return status, handle
def test_stream_after_warmup(self): nodeA = self.cluster.servers[0] bucket = self.bucket_util.buckets[1] shell_conn = RemoteMachineShellConnection(nodeA) cb_stat_obj = Cbstats(shell_conn) # load all buckets doc_gen = doc_generator(self.key, 0, self.num_items) self._load_all_buckets(self.master, doc_gen, "create", 0) self._wait_for_stats_all_buckets() # store expected vb seqnos originalVbInfo = self.all_vb_info(cb_stat_obj, bucket.name) # restart node self.assertTrue(self.stop_node(0), msg="Failed during stop_node") self.sleep(5, "Wait after stop_node") self.assertTrue(self.start_node(0), msg="Failed during start_node") rest = RestHelper(RestConnection(nodeA)) self.assertTrue(rest.is_ns_server_running(), msg="Failed while is_ns_server_running") self.sleep(2, "Wait after ns_server_start") # verify original vbInfo can be streamed dcp_client = self.dcp_client(nodeA, PRODUCER, bucket_name=bucket.name) for vbucket in originalVbInfo: vb_uuid, _, high_seqno = originalVbInfo[vbucket] stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, vb_uuid) _ = stream.run() self.assertTrue(high_seqno == stream.last_by_seqno, msg="Mismatch in high_seqno. {0} == {1}".format( high_seqno, stream.last_by_seqno)) # Disconnect the shell_conn shell_conn.disconnect()
def rebalance_in_with_warming_up(self): servs_in = self.servers[self.nodes_init:self.nodes_init + self.nodes_in] servs_init = self.servers[:self.nodes_init] warmup_node = servs_init[-1] shell = RemoteMachineShellConnection(warmup_node) shell.stop_couchbase() self.sleep(20) shell.start_couchbase() shell.disconnect() try: rebalance = self.cluster.async_rebalance(servs_init, servs_in, []) rebalance.result() except RebalanceFailedException: self.log.info("rebalance was failed as expected") self.assertTrue(ClusterOperationHelper._wait_warmup_completed(self, [warmup_node], \ self.default_bucket_name, wait_time=self.wait_timeout * 10)) self.log.info("second attempt to rebalance") rebalance = self.cluster.async_rebalance(servs_init + servs_in, [], []) rebalance.result() self.verify_cluster_stats(self.servers[:self.nodes_in + self.nodes_init])
def execute_statement_on_cbas(self, statement, server, mode=None): """ Executes a statement on CBAS using the REST API through curl command """ shell = RemoteMachineShellConnection(server) if mode: output, error = shell.execute_command( """curl -s --data pretty=true --data mode={2} --data-urlencode 'statement={1}' http://{0}:8095/analytics/service -v""" .format(self.cbas_node.ip, statement, mode)) else: output, error = shell.execute_command( """curl -s --data pretty=true --data-urlencode 'statement={1}' http://{0}:8095/analytics/service -v""" .format(self.cbas_node.ip, statement)) response = "" for line in output: response = response + line response = json.loads(response) self.log.info(response) shell.disconnect() if "errors" in response: errors = response["errors"] else: errors = None if "results" in response: results = response["results"] else: results = None if "handle" in response: handle = response["handle"] else: handle = None return response["status"], response["metrics"], errors, results, handle
def initialize(self, params): start_time = time.time() cluster_initialized = False server = params["server"] while time.time() < (start_time + (5 * 60)): rest = RestConnection(server) try: if server.data_path: remote_client = RemoteMachineShellConnection(server) remote_client.execute_command('rm -rf {0}/*'.format(server.data_path)) # Make sure that data_path is writable by membase user remote_client.execute_command("chown -R membase.membase {0}".format(server.data_path)) remote_client.disconnect() rest.set_data_path(data_path=server.data_path) rest.init_cluster(username=server.rest_username, password=server.rest_password) rest.init_cluster_memoryQuota(memoryQuota=rest.get_nodes_self().mcdMemoryReserved) cluster_initialized = True break except ServerUnavailableException: log.error("error happened while initializing the cluster @ {0}".format(server.ip)) log.info('sleep for 5 seconds before trying again ...') time.sleep(5) if not cluster_initialized: raise Exception("unable to initialize membase node")
def testSettingCompacttion(self): '''setting-compacttion OPTIONS: --compaction-db-percentage=PERCENTAGE at which point database compaction is triggered --compaction-db-size=SIZE[MB] at which point database compaction is triggered --compaction-view-percentage=PERCENTAGE at which point view compaction is triggered --compaction-view-size=SIZE[MB] at which point view compaction is triggered --compaction-period-from=HH:MM allow compaction time period from --compaction-period-to=HH:MM allow compaction time period to --enable-compaction-abort=[0|1] allow compaction abort when time expires --enable-compaction-parallel=[0|1] allow parallel compaction for database and view''' compaction_db_percentage = self.input.param("compaction-db-percentage", None) compaction_db_size = self.input.param("compaction-db-size", None) compaction_view_percentage = self.input.param("compaction-view-percentage", None) compaction_view_size = self.input.param("compaction-view-size", None) compaction_period_from = self.input.param("compaction-period-from", None) compaction_period_to = self.input.param("compaction-period-to", None) enable_compaction_abort = self.input.param("enable-compaction-abort", None) enable_compaction_parallel = self.input.param("enable-compaction-parallel", None) bucket = self.input.param("bucket", "default") output = self.input.param("output", '') remote_client = RemoteMachineShellConnection(self.master) cli_command = "setting-compaction" options = (" --compaction-db-percentage={0}".format(compaction_db_percentage), "")[compaction_db_percentage is None] options += (" --compaction-db-size={0}".format(compaction_db_size), "")[compaction_db_size is None] options += (" --compaction-view-percentage={0}".format(compaction_view_percentage), "")[compaction_view_percentage is None] options += (" --compaction-view-size={0}".format(compaction_view_size), "")[compaction_view_size is None] options += (" --compaction-period-from={0}".format(compaction_period_from), "")[compaction_period_from is None] options += (" --compaction-period-to={0}".format(compaction_period_to), "")[compaction_period_to is None] options += (" --enable-compaction-abort={0}".format(enable_compaction_abort), "")[enable_compaction_abort is None] options += (" --enable-compaction-parallel={0}".format(enable_compaction_parallel), "")[enable_compaction_parallel is None] output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, options=options, cluster_host="localhost", user=self.ldapUser, password=self.ldapPass) expectedResults = {"parallel_db_and_view_compaction":False, "database_fragmentation_threshold:size":10485760, "database_fragmentation_threshold:view_fragmentation_threshold":{}, "real_userid:source":self.source, "real_userid:user":self.ldapUser, "remote:ip":"::1", "remote:port":60019} self.checkConfig(8225, self.master, expectedResults) remote_client.disconnect()
def validate_backup_create(self): """ Validates that the backup directory is created as expected Validates the backup metadata using backup-meta.json :return: status and message """ remote_client = RemoteMachineShellConnection( self.backupset.backup_host) info = remote_client.extract_remote_info().type.lower() if info == 'linux' or info == 'mac': command = "ls -R {0}/{1}".format(self.backupset.directory, self.backupset.name) o, e = remote_client.execute_command(command) elif info == 'windows': o = remote_client.list_files("{0}/{1}".format( self.backupset.directory, self.backupset.name)) if not o and len(o) != 2 and o[1] != "backup-meta.json": return False, "Backup create did not create backup-meta file." remote_client.disconnect() files_validations = BackupRestoreFilesValidations(self.backupset) status, msg = files_validations.validate_backup_meta_json() if status: msg += "\nBackup create validation success." return status, msg
def test_max_ttl_bucket(self): """ From vulcan, EE bucket has has an option to set --max-ttl, not it CE. This test is make sure CE could not create bucket with option --max-ttl This test must pass default_bucket=False """ if self.cb_version[:5] not in COUCHBASE_FROM_VULCAN: self.log.info("This test only for vulcan and later") return cmd = 'curl -X POST -u Administrator:password \ http://{0}:8091/pools/default/buckets \ -d name=bucket0 \ -d maxTTL=100 \ -d ramQuotaMB=100 '.format(self.master.ip) if self.cli_test: cmd = "{0}couchbase-cli bucket-create -c {1}:8091 --username Administrator \ --password password --bucket bucket0 --bucket-type couchbase \ --bucket-ramsize 512 --bucket-replica 1 --bucket-priority high \ --bucket-eviction-policy fullEviction --enable-flush 0 \ --enable-index-replica 1 --max-ttl 200".format( self.bin_path, self.master.ip) conn = RemoteMachineShellConnection(self.master) output, error = conn.execute_command(cmd) conn.log_command_output(output, error) mesg = "Max TTL is supported in enterprise edition only" if self.cli_test: mesg = "Maximum TTL can only be configured on enterprise edition" if output and mesg not in str(output[0]): self.fail("max ttl feature should not in Community Edition") buckets = RestConnection(self.master).get_buckets() if buckets: for bucket in buckets: self.log.info("bucekt in cluser: {0}".format(bucket.name)) if bucket.name == "bucket0": self.fail("Failed to enforce feature max ttl in CE.") conn.disconnect()
def test_large_file_version(self): rest = RestConnection(self.cluster.master) remote_client = RemoteMachineShellConnection(self.cluster.master) remote_client.extract_remote_info() self._load_all_buckets(self.gen_load, "create") self.disable_compaction() self._monitor_DB_fragmentation() # rename here remote_client.stop_couchbase() time.sleep(5) remote_client.execute_command( "cd /opt/couchbase/var/lib/couchbase/data/default;rename .1 .65535 *.1" ) remote_client.execute_command( "cd /opt/couchbase/var/lib/couchbase/data/default;rename .2 .65535 *.2" ) remote_client.start_couchbase() for _ in range(5): self.log.info("starting a compaction iteration") compaction_task = self.cluster.async_compact_bucket( self.cluster.master, self.default_bucket_name) compact_run = remote_client.wait_till_compaction_end( rest, self.default_bucket_name, timeout_in_seconds=self.wait_timeout) res = compaction_task.result(self.wait_timeout) if compact_run: self.log.info("auto compaction run successfully") else: self.fail("auto compaction does not run") remote_client.disconnect()
def eventing_operation(self, subcommand, function_name, boundary="from-everything", file_name=None, name=True): options = self._get_default_options() options += " --name " + function_name if boundary: options += " --boundary " + boundary if name: options += " --file " + file_name remote_client = RemoteMachineShellConnection(self.server) stdout, stderr = remote_client.couchbase_cli(subcommand, self.hostname, options) remote_client.disconnect() if subcommand == "import": return stdout, stderr, self._was_success(stdout, "Events imported") elif subcommand == "deploy": return stdout, stderr, self._was_success(stdout, "Request to deploy the function was accepted") elif subcommand == "pause": return stdout, stderr, self._was_success(stdout, "Function was paused") elif subcommand == "resume": return stdout, stderr, self._was_success(stdout, "Function was resumed") elif subcommand == "undeploy": return stdout, stderr, self._was_success(stdout, "Request to undeploy the function was accepted") elif subcommand == "delete": return stdout, stderr, self._was_success(stdout, "Request to undeploy the function was accepted") elif subcommand == "list": return stdout, stderr
def test_start_stop_DB_compaction(self): rest = RestConnection(self.master) remote_client = RemoteMachineShellConnection(self.master) self.log.info('loading the buckets') self._load_all_buckets(self.master, self.gen_load, "create", 0, 1) self.log.info('disabling compaction') self.disable_compaction() self.log.info('monitor db fragmentation') self._monitor_DB_fragmentation() self.log.info('async compact the bucket') compaction_task = self.cluster.async_compact_bucket(self.master, self.default_bucket_name) self.log.info('cancel bucket compaction') self._cancel_bucket_compaction(rest, self.default_bucket_name) #compaction_task.result(self.wait_timeout) self.log.info('compact again') self.cluster.async_compact_bucket(self.master, self.default_bucket_name) self.log.info('waiting for compaction to end') compact_run = remote_client.wait_till_compaction_end(rest, self.default_bucket_name, timeout_in_seconds=self.wait_timeout) if compact_run: self.log.info("auto compaction run successfully") else: self.fail("auto compaction does not run") remote_client.disconnect()
def rebalance_out_with_warming_up(self): master_restart = self.input.param("master_restart", False) if master_restart: warmup_node = self.cluster.master else: warmup_node = self.cluster.servers[len(self.cluster.servers) - self.nodes_out - 1] servs_out = self.cluster.servers[len(self.cluster.servers) - self.nodes_out:] shell = RemoteMachineShellConnection(warmup_node) shell.stop_couchbase() self.sleep(20) shell.start_couchbase() shell.disconnect() try: rebalance = self.task.async_rebalance(self.cluster.servers, [], servs_out) self.task.jython_task_manager.get_task_result(rebalance) self.cluster.nodes_in_cluster = list( set(self.cluster.nodes_in_cluster) - set(servs_out)) except RebalanceFailedException: self.log.info("rebalance was failed as expected") self.assertTrue( self.bucket_util._wait_warmup_completed( self, [warmup_node], 'default', wait_time=self.wait_timeout * 10)) self.log.info("second attempt to rebalance") rebalance = self.task.async_rebalance(self.cluster.servers, [], servs_out) self.task.jython_task_manager.get_task_result(rebalance) self.cluster.nodes_in_cluster = list( set(self.cluster.nodes_in_cluster) - set(servs_out)) self.bucket_util.verify_cluster_stats(self.num_items) self.bucket_util.verify_unacked_bytes_all_buckets()
def disable_IPV6_grub_level(self): """ Disable IPV6 at grub level for all nodes in the cluster """ for server in self.servers: shell = RemoteMachineShellConnection(server) shell.execute_command( '''sed -i 's/ipv6.disable=0 //; s/ipv6.disable=1 //; s/GRUB_CMDLINE_LINUX="/GRUB_CMDLINE_LINUX="ipv6.disable=1 /' /etc/default/grub''' ) shell.execute_command("grub2-mkconfig -o /boot/grub2/grub.cfg") shell.reboot_node() time.sleep(10) shell = RemoteMachineShellConnection(server) output, error = shell.execute_command("ifconfig | grep inet6") if output == [] and error == []: log.info("IPv6 Successfully Disabled for {0}".format( server.ip)) else: log.info("Cant disable IPv6") log.info( "Output message is {0} and error message is {1}".format( output, error)) output, error = shell.execute_command("iptables -F") shell.disconnect()
def testInfoCommands(self): remote_client = RemoteMachineShellConnection(self.master) cli_command = "server-list" output, error = remote_client.execute_couchbase_cli( cli_command=cli_command, cluster_host="localhost", user="******", password="******") server_info = self._get_cluster_info(remote_client) result = server_info["otpNode"] + " " + server_info[ "hostname"] + " " + server_info["status"] + " " + server_info[ "clusterMembership"] self.assertEqual( result, "ns_1@{0} {0}:8091 healthy active".format("127.0.0.1")) cli_command = "bucket-list" output, error = remote_client.execute_couchbase_cli( cli_command=cli_command, cluster_host="localhost", user="******", password="******") self.assertEqual([], output) remote_client.disconnect()
def test_rollback_and_persistence_race_condition(self): cluster = self.cluster gen_load = doc_generator(self.key, 0, self.num_items) for bucket in self.bucket_util.buckets: task = self.task.async_load_gen_docs( self.cluster, bucket, gen_load, "create", 0, batch_size=10, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries) self.task.jython_task_manager.get_task_result(task) # Stop persistence for server in cluster.servers[:self.nodes_init]: # Create cbepctl command object node_shell_conn = RemoteMachineShellConnection(server) cbepctl_obj = Cbepctl(node_shell_conn) for bucket in self.bucket_util.buckets: cbepctl_obj.persistence(bucket.name, "stop") # Disconnect the shell_connection node_shell_conn.disconnect() self.sleep(10, "Wait after stop_persistence") # more (non-intersecting) load gen_load = doc_generator(self.key, 0, self.num_items, doc_size=64) for bucket in self.bucket_util.buckets: task = self.task.async_load_gen_docs( self.cluster, bucket, gen_load, "create", 0, batch_size=10, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries) self.task.jython_task_manager.get_task_result(task) shell = RemoteMachineShellConnection(cluster.servers[0]) shell.kill_memcached() self.sleep(10, "Wait after kill memcached") node1_shell_conn = RemoteMachineShellConnection(cluster.servers[0]) node2_shell_conn = RemoteMachineShellConnection(cluster.servers[1]) node1_cb_stat_obj = Cbstats(node1_shell_conn) node2_cb_stat_obj = Cbstats(node2_shell_conn) node1_items = node1_cb_stat_obj.all_stats(bucket, "curr_items_tot") node2_items = node2_cb_stat_obj.all_stats(bucket, "curr_items_tot") # Disconnect the opened connections node1_shell_conn.disconnect() node2_shell_conn.disconnect() self.assertTrue(node1_items == node2_items, 'Node items not equal. Node 1:{0}, node 2:{1}' .format(node1_items, node2_items))