def set_metadata_purge_interval(self, value, buckets=[], node=None): self.log.info( "Changing the bucket properties by changing {0} to {1}".format( "purge_interval", value)) if not buckets: buckets = self.buckets if node is None: node = self.cluster.master rest = RestConnection(node) shell = RemoteMachineShellConnection(node) shell.enable_diag_eval_on_non_local_hosts() shell.disconnect() for bucket in buckets: cmd = '{ok, BC} = ns_bucket:get_bucket(' \ '"%s"), BC2 = lists:keyreplace(purge_interval, ' \ '1, BC, {purge_interval, %f})' \ ', ns_bucket:set_bucket_config("%s", BC2).' \ % (bucket.name, value, bucket.name) rest.diag_eval(cmd) # Restart Memcached in all cluster nodes to reflect the settings for server in self.cluster_util.get_kv_nodes(master=node): shell = RemoteMachineShellConnection(server) shell.restart_couchbase() shell.disconnect() # Check bucket-warm_up after Couchbase restart retry_count = 10 buckets_warmed_up = self.bucket_util.is_warmup_complete( buckets, retry_count) if not buckets_warmed_up: self.log.critical("Few bucket(s) not warmed up " "within expected time")
def _induce_rebalance_test_condition(self, test_failure_condition): if test_failure_condition == "verify_replication": set_command = "testconditions:set(verify_replication, {fail, \"" + "default" + "\"})" elif test_failure_condition == "backfill_done": set_command = "testconditions:set(backfill_done, {for_vb_move, \"" + "default\", 1 , " + "fail})" else: set_command = "testconditions:set({0}, fail)".format( test_failure_condition) get_command = "testconditions:get({0})".format(test_failure_condition) for server in self.servers: rest = RestConnection(server) shell = RemoteMachineShellConnection(server) shell.enable_diag_eval_on_non_local_hosts() _, content = rest.diag_eval(set_command) self.log.debug("Set Command : {0} Return : {1}".format( set_command, content)) shell.disconnect() for server in self.servers: rest = RestConnection(server) shell = RemoteMachineShellConnection(server) shell.enable_diag_eval_on_non_local_hosts() _, content = rest.diag_eval(get_command) self.log.info("Command : {0} Return : {1}".format( get_command, content))
def update_master(self, node_in_cluster=None): """ Update the master of the cluster with the orchestrator of the cluster. :param node_in_cluster: Any node that is still part of the cluster. Note that we need to enable diag_eval on non local hosts first before running this method. :return: """ if node_in_cluster is None: node_in_cluster = self.master # Enable diag_eval outside localhost shell = RemoteMachineShellConnection(node_in_cluster) shell.enable_diag_eval_on_non_local_hosts() shell.disconnect() rest = RestConnection(node_in_cluster) command = "mb_master:master_node()." status, content = rest.diag_eval(command) master_ip = content.split("@")[1].replace("\\", '').replace( "'", "") self.master = [server for server in self.servers if server.ip == master_ip][0] return status, content
def test_multiple_backups_merges(self): self.log.info("*** start to load items to all buckets") self.expected_error = self.input.param("expected_error", None) if int(self.active_resident_threshold) > 0: self.log.info("Disable compaction to speed up dgm") RestConnection(self.master).disable_auto_compaction() if self.expires: for bucket in self.buckets: cb = self._get_python_sdk_client(self.master.ip, bucket, self.backupset.cluster_host) for i in range(1, self.num_items + 1): cb.upsert("doc" + str(i), {"key": "value"}) else: self._load_all_buckets(self.master, self.initial_load_gen, "create", self.expires) self.log.info("*** done to load items to all buckets") self.backup_create_validate() for i in range(1, self.number_of_repeats + 1): self.do_backup_merge_actions() start = self.number_of_backups_taken end = self.number_of_backups_taken if self.backup_corrupted: self.log.info( "Stop restore due to backup files corrupted as intended") return if self.reset_restore_cluster: self.log.info("*** start to reset cluster") self.backup_reset_clusters(self.cluster_to_restore) if self.same_cluster: self._initialize_nodes(Cluster(), self.servers[:self.nodes_init]) else: shell = RemoteMachineShellConnection(self.input.clusters[0][0]) shell.enable_diag_eval_on_non_local_hosts() shell.disconnect() rest = RestConnection(self.input.clusters[0][0]) rest.force_eject_node() master_services = self.get_services( [self.backupset.cluster_host], self.services_init, start_node=0) info = rest.get_nodes_self() if info.memoryQuota and int(info.memoryQuota) > 0: self.quota = info.memoryQuota rest.init_node() self.log.info("Done reset cluster") self.sleep(10) """ Add built-in user cbadminbucket to second cluster """ self.add_built_in_server_user( node=self.input.clusters[0][:self.nodes_init][0]) self.backupset.start = start self.backupset.end = end self.log.info("*** start restore validation") self.backup_restore_validate(compare_uuid=False, seqno_compare_function=">=", expected_error=self.expected_error)
def delete_rebalance_test_condition(self, test_failure_condition): delete_command = "testconditions:delete({0})".format(test_failure_condition) for server in self.servers: rest = RestConnection(server) shell = RemoteMachineShellConnection(server) shell.enable_diag_eval_on_non_local_hosts() _, content = rest.diag_eval(delete_command) self.log.debug("Delete Command : {0} Return : {1}".format(delete_command, content)) shell.disconnect()
def setUp(self): super(CBTransferTests, self).setUp() self.origin_buckets = list(self.buckets) self.master = self.server_recovery self.add_built_in_server_user(node=self.master) shell = RemoteMachineShellConnection(self.master) shell.enable_diag_eval_on_non_local_hosts() shell.disconnect() self._bucket_creation() self.buckets = list(self.origin_buckets)
def _delete_rebalance_test_condition(self, test_failure_condition): if test_failure_condition.startswith("delay_"): test_failure_condition = test_failure_condition[6:] delete_command = "testconditions:delete(%s)" % test_failure_condition get_command = "testconditions:get(%s)" % test_failure_condition for server in self.servers: rest = RestConnection(server) shell = RemoteMachineShellConnection(server) shell.enable_diag_eval_on_non_local_hosts() _, content = rest.diag_eval(delete_command) self.log.info("Command: %s, Return: %s" % (delete_command, content)) for server in self.servers: rest = RestConnection(server) shell = RemoteMachineShellConnection(server) shell.enable_diag_eval_on_non_local_hosts() _, content = rest.diag_eval(get_command) self.log.info("Command: %s, Return: %s" % (get_command, content))
def check_for_panic_and_mini_dumps(self, servers): panic_str = "panic" panic_count = 0 for server in servers: shell = RemoteMachineShellConnection(server) output, error = shell.enable_diag_eval_on_non_local_hosts() if output is not None: if "ok" not in output: self.log.error( "Error in enabling diag/eval on non-local hosts on {}: Error: {}" .format(server.ip, error)) else: self.log.debug( "Enabled diag/eval for non-local hosts from {}".format( server.ip)) else: self.log.debug( "Running in compatibility mode, not enabled diag/eval for non-local hosts" ) _, dir_name = RestConnection(server).diag_eval( 'filename:absname(element(2, application:get_env(ns_server,error_logger_mf_dir))).' ) log = str(dir_name) + '/*' count, err = shell.execute_command( "zgrep \"{0}\" {1} | wc -l".format(panic_str, log)) if isinstance(count, list): count = int(count[0]) else: count = int(count) if count > panic_count: self.log.info( "===== PANIC OBSERVED IN THE LOGS ON SERVER {0}=====". format(server.ip)) panic_trace, _ = shell.execute_command( "zgrep \"{0}\" {1}".format(panic_str, log)) self.log.info("\n {0}".format(panic_trace)) panic_count = count os_info = shell.extract_remote_info() if os_info.type.lower() == "windows": # This is a fixed path in all windows systems inside couchbase dir_name_crash = 'c://CrashDumps' else: dir_name_crash = str(dir_name) + '/../crash/' core_dump_count, err = shell.execute_command( "ls {0}| wc -l".format(dir_name_crash)) if isinstance(core_dump_count, list): core_dump_count = int(core_dump_count[0]) else: core_dump_count = int(core_dump_count) if core_dump_count > 0: self.log.info( "===== CORE DUMPS SEEN ON SERVER {0} : {1} crashes seen =====" .format(server.ip, core_dump_count)) shell.disconnect()
def _induce_rebalance_test_condition(self, test_failure_condition, bucket_name="default", vb_num=1, delay_time=60000): if test_failure_condition == "verify_replication": set_command = 'testconditions:set(verify_replication, ' \ '{fail, "%s"})' % bucket_name elif test_failure_condition == "backfill_done": set_command = 'testconditions:set(backfill_done, ' \ '{for_vb_move, "%s", %s , fail})' \ % (bucket_name, vb_num) elif test_failure_condition == "delay_rebalance_start": set_command = 'testconditions:set(rebalance_start, {delay, %s}).' \ % delay_time elif test_failure_condition == "delay_verify_replication": set_command = 'testconditions:set(verify_replication, ' \ '{delay, "%s", %s})' % (bucket_name, delay_time) elif test_failure_condition == "delay_backfill_done": set_command = 'testconditions:set(backfill_done, ' \ '{for_vb_move, "%s", %s, {delay, %s}})' \ % (bucket_name, vb_num, delay_time) else: set_command = "testconditions:set(%s, fail)" \ % test_failure_condition get_command = "testconditions:get(%s)" % test_failure_condition for server in self.servers: rest = RestConnection(server) shell = RemoteMachineShellConnection(server) shell.enable_diag_eval_on_non_local_hosts() _, content = rest.diag_eval(set_command) self.log.debug("Set Command: %s. Return: %s" % (set_command, content)) shell.disconnect() for server in self.servers: rest = RestConnection(server) shell = RemoteMachineShellConnection(server) shell.enable_diag_eval_on_non_local_hosts() _, content = rest.diag_eval(get_command) self.log.info("Command: %s, Return: %s" % (get_command, content))
def setUp(self): for server in TestInputSingleton.input.servers: remote = RemoteMachineShellConnection(server) remote.enable_diag_eval_on_non_local_hosts() remote.disconnect() super(AlternateAddressTests, self).setUp() self.remove_all_alternate_address_settings() self.cluster_helper = Cluster() self.ex_path = self.tmp_path + "export{0}/".format(self.master.ip) self.num_items = self.input.param("items", 1000) self.client_os = self.input.param("client_os", "linux") self.localhost = self.input.param("localhost", False) self.json_create_gen = JsonDocGenerator("altaddr", op_type="create", encoding="utf-8", start=0, end=self.num_items) self.json_delete_gen = JsonDocGenerator("imex", op_type="delete", encoding="utf-8", start=0, end=self.num_items)
def enable_diag_eval_on_non_local_hosts(self, master): """ Enable diag/eval to be run on non-local hosts. :param master: Node information of the master node of the cluster :return: Nothing """ remote = RemoteMachineShellConnection(master) output, error = remote.enable_diag_eval_on_non_local_hosts() if "ok" not in output: self.log.error("Error in enabling diag/eval on non-local hosts on {}. {}".format(master.ip, output)) raise Exception("Error in enabling diag/eval on non-local hosts on {}".format(master.ip)) else: self.log.info("Enabled diag/eval for non-local hosts from {}".format(master.ip))
def setUp(self): self.input = TestInputSingleton.input self.case_number = self.input.param("case_number", 0) self.use_master = self.input.param("use_master", False) self.skip_services = self.input.param("skip_services", True) self.replicas = self.input.param("replicas", 1) self.servers = self.input.servers self.log = logger.Logger().get_logger() self.master = self.servers[0] self.rest = RestConnection(self.master) self.timeout = 60 self.loaded_items = dict() AutoReprovisionBaseTest.common_setup(self.input, self) self._cluster_setup() if self.use_master: self.server_fail = self.servers[0] self.master = self.servers[1] else: self.server_fail = self.servers[1] for server in self.servers: remote = RemoteMachineShellConnection(server) output, error = remote.enable_diag_eval_on_non_local_hosts()
def setUp(self): super(EnterpriseBKRSNewBaseTest, self).setUp() self.clusters = self.get_clusters() self.master = self.servers[0] self.task = self.get_task() self.taskmgr = self.get_task_mgr() self.backupset = Backupset() self.cmd_ext = "" self.should_fail = self.input.param("should-fail", False) self.restore_should_fail = self.input.param("restore_should_fail", False) self.merge_should_fail = self.input.param("merge_should_fail", False) self.database_path = Linux.COUCHBASE_DATA_PATH cmd = 'curl -g {0}:8091/diag/eval -u {1}:{2} '.format( self.master.ip, self.master.rest_username, self.master.rest_password) cmd += '-d "path_config:component_path(bin)."' shell = RemoteMachineShellConnection(self.master) output, error = shell.enable_diag_eval_on_non_local_hosts() shell.disconnect() bin_path = subprocess.check_output(cmd, shell=True) if not self.skip_init_check_cbserver: if "bin" not in bin_path: self.fail("Check if cb server install on %s" % self.master.ip) else: self.cli_command_location = bin_path.replace('"', '') + "/" self.debug_logs = self.input.param("debug-logs", False) self.backupset.directory = self.input.param("dir", "/tmp/entbackup") self.backupset.user_env = self.input.param("user-env", False) self.backupset.passwd_env = self.input.param("passwd-env", False) self.backupset.log_archive_env = self.input.param( "log-archive-env", False) self.backupset.log_redaction = self.input.param("log-redaction", False) self.backupset.redaction_salt = self.input.param( "redaction-salt", None) self.backupset.no_log_output_flag = self.input.param( "no-log-output-flag", False) self.backupset.ex_logs_path = self.input.param("ex-logs-path", None) self.backupset.overwrite_user_env = self.input.param( "overwrite-user-env", False) self.backupset.overwrite_passwd_env = self.input.param( "overwrite-passwd-env", False) self.backupset.disable_conf_res_restriction = self.input.param( "disable-conf-res-restriction", None) self.backupset.force_updates = self.input.param("force-updates", False) self.backupset.resume = self.input.param("resume", False) self.backupset.purge = self.input.param("purge", False) self.backupset.start = self.input.param("start", 1) self.backupset.end = self.input.param("stop", 1) self.backupset.number_of_backups = self.input.param( "number_of_backups", 1) self.replace_ttl_with = self.input.param("replace-ttl-with", None) self.backupset.backup_host = self.servers[-1] self.backupset.name = self.input.param("name", "backup") self.backupset.filter_keys = self.input.param("filter-keys", "") self.backupset.random_keys = self.input.param("random_keys", False) self.backupset.filter_values = self.input.param("filter-values", "") self.backupset.no_ssl_verify = self.input.param("no-ssl-verify", False) self.backupset.secure_conn = self.input.param("secure-conn", False) self.backupset.bk_no_cert = self.input.param("bk-no-cert", False) self.backupset.rt_no_cert = self.input.param("rt-no-cert", False) self.backupset.backup_list_name = self.input.param("list-names", None) self.backupset.backup_incr_backup = self.input.param( "incr-backup", None) self.backupset.bucket_backup = self.input.param("bucket-backup", None) shell = RemoteMachineShellConnection(self.servers[0]) info = shell.extract_remote_info().type.lower() self.root_path = Linux.ROOT_PATH self.wget = "wget" self.os_name = "linux" self.tmp_path = "/tmp/" self.long_help_flag = "--help" self.short_help_flag = "-h" self.cygwin_bin_path = "" self.enable_firewal = False self.rfc3339_date = "date +%s --date='{0} seconds' | ".format(self.replace_ttl_with) + \ "xargs -I {} date --date='@{}' --rfc-3339=seconds | "\ "sed 's/ /T/'" self.seconds_with_ttl = "date +%s --date='{0} seconds'".format( self.replace_ttl_with) if info == Linux.NAME: if self.nonroot: base_path = "/home/{0}".format(self.master.ssh_username) self.database_path = "{0}{1}".format(base_path, Linux.COUCHBASE_DATA_PATH) self.root_path = "/home/{0}/".format(self.master.ssh_username) elif info == Windows.NAME: self.os_name = Windows.NAME self.cmd_ext = ".exe" self.wget = "/cygdrive/c/automation/wget.exe" self.database_path = Windows.COUCHBASE_DATA_PATH_RAW self.root_path = Windows.ROOT_PATH self.tmp_path = Windows.TMP_PATH self.long_help_flag = "help" self.short_help_flag = "h" self.cygwin_bin_path = Windows.CYGWIN_BIN_PATH self.rfc3339_date = "date +%s --date='{0} seconds' | ".format(self.replace_ttl_with) + \ "{0}xargs -I {{}} date --date=\"@'{{}}'\" --rfc-3339=seconds | "\ .format(self.cygwin_bin_path) + \ "sed 's/ /T/'" win_format = "C:/Program Files" cygwin_format = "/cygdrive/c/Program\ Files" if win_format in self.cli_command_location: self.cli_command_location = self.cli_command_location.replace( win_format, cygwin_format) self.backupset.directory = self.input.param( "dir", Windows.TMP_PATH_RAW + "entbackup") elif info == 'mac': self.backupset.directory = self.input.param( "dir", "/tmp/entbackup") else: raise Exception("OS not supported.") self.non_master_host = self.input.param("non-master", False) self.value_size = self.input.param("value_size", 512) self.no_progress_bar = self.input.param("no-progress-bar", True) self.multi_threads = self.input.param("multi_threads", False) self.threads_count = self.input.param("threads_count", 1) self.bucket_delete = self.input.param("bucket_delete", False) self.bucket_flush = self.input.param("bucket_flush", False) self.commit = self.input.param("commit", True) self.ops_type = self.input.param("ops_type", "create") self.num_threads = self.input.param("num_threads", 5) self.bk_with_ttl = self.input.param("bk-with-ttl", None) self.create_fts_index = self.input.param("create-fts-index", False) self.reset_restore_cluster = self.input.param("reset-restore-cluster", False) self.backupset.user_env_with_prompt = \ self.input.param("user-env-with-prompt", False) self.backupset.passwd_env_with_prompt = \ self.input.param("passwd-env-with-prompt", False) self.restore_compression_mode = self.input.param( "restore-compression-mode", None) self.force_version_upgrade = self.input.param("force-version-upgrade", None) self.skip_buckets = self.input.param("skip_buckets", False) self.num_replicas = self.input.param("replicas", 2) self.restore_only = self.input.param("restore-only", False) if self.non_master_host: self.backupset.cluster_host = self.servers[1] self.backupset.cluster_host_username = self.servers[ 1].rest_username self.backupset.cluster_host_password = self.servers[ 1].rest_password else: self.backupset.cluster_host = self.servers[0] self.backupset.cluster_host_username = self.servers[ 0].rest_username self.backupset.cluster_host_password = self.servers[ 0].rest_password self.same_cluster = self.input.param("same-cluster", False) if self.same_cluster: self.backupset.restore_cluster_host = self.input.clusters[0][0] self.backupset.restore_cluster_host_username = self.input.clusters[ 0][0].rest_username self.backupset.restore_cluster_host_password = self.input.clusters[ 0][0].rest_password else: self.backupset.restore_cluster_host = self.input.clusters[1][0] self.backupset.restore_cluster_host_username = self.input.clusters[ 1][0].rest_username self.backupset.restore_cluster_host_password = self.input.clusters[ 1][0].rest_password """ new user to test RBAC """ self.cluster_new_user = self.input.param("new_user", None) if self.cluster_new_user: self.backupset.cluster_host_username = self.cluster_new_user self.backupset.restore_cluster_host_username = self.cluster_new_user self.backups = [] self.number_of_backups_taken = 0 for cluster in self.clusters: self.cluster_util.add_all_nodes_then_rebalance( cluster, cluster.servers[1:])
def test_backup_restore_collection_sanity(self): """ 1. Create default bucket on the cluster and loads it with given number of items 2. Perform updates and create backups for specified number of times (test param number_of_backups) 3. Perform restores for the same number of times with random start and end values """ self.log.info("*** create collection in all buckets") self.log.info("*** start to load items to all buckets") self.active_resident_threshold = 100 self.load_all_buckets(self.backupset.cluster_host) self.log.info("*** done to load items to all buckets") self.ops_type = self.input.param("ops-type", "update") self.expected_error = self.input.param("expected_error", None) self.create_scope_cluster_host() self.create_collection_cluster_host(self.backupset.col_per_scope) backup_scopes = self.get_bucket_scope_cluster_host() backup_collections = self.get_bucket_collection_cluster_host() col_stats = self.get_collection_stats_cluster_host() for backup_scope in backup_scopes: bk_scope_id = self.get_scopes_id_cluster_host(backup_scope) if self.auto_failover: self.log.info("Enabling auto failover on " + str(self.backupset.cluster_host)) rest_conn = RestConnection(self.backupset.cluster_host) rest_conn.update_autofailover_settings(self.auto_failover, self.auto_failover_timeout) self.backup_create_validate() for i in range(1, self.backupset.number_of_backups + 1): if self.ops_type == "update": self.log.info("*** start to update items in all buckets") self.load_all_buckets(self.backupset.cluster_host, ratio=0.1) self.log.info("*** done update items in all buckets") self.sleep(10) self.log.info("*** start to validate backup cluster") self.backup_cluster_validate() self.targetMaster = True start = randrange(1, self.backupset.number_of_backups + 1) if start == self.backupset.number_of_backups: end = start else: end = randrange(start, self.backupset.number_of_backups + 1) self.log.info("*** start to restore cluster") restored = {"{0}/{1}".format(start, end): ""} for i in range(1, self.backupset.number_of_backups + 1): if self.reset_restore_cluster: self.log.info("*** start to reset cluster") self.backup_reset_clusters(self.cluster_to_restore) if self.same_cluster: self._initialize_nodes(Cluster(), self.servers[:self.nodes_init]) else: shell = RemoteMachineShellConnection( self.backupset.restore_cluster_host) shell.enable_diag_eval_on_non_local_hosts() shell.disconnect() rest = RestConnection(self.backupset.restore_cluster_host) rest.force_eject_node() rest.init_node() self.log.info("Done reset cluster") self.sleep(10) """ Add built-in user cbadminbucket to second cluster """ self.add_built_in_server_user( node=self.input.clusters[0][:self.nodes_init][0]) self.backupset.start = start self.backupset.end = end self.log.info("*** start restore validation") data_map_collection = [] for scope in backup_scopes: if "default" in scope: continue data_map_collection.append(self.buckets[0].name + "." + scope + "=" + \ self.buckets[0].name + "." + scope) self.bucket_map_collection = ",".join(data_map_collection) self.backup_restore_validate(compare_uuid=False, seqno_compare_function=">=", expected_error=self.expected_error) if self.backupset.number_of_backups == 1: continue while "{0}/{1}".format(start, end) in restored: start = randrange(1, self.backupset.number_of_backups + 1) if start == self.backupset.number_of_backups: end = start else: end = randrange(start, self.backupset.number_of_backups + 1) restored["{0}/{1}".format(start, end)] = "" restore_scopes = self.get_bucket_scope_restore_cluster_host() restore_collections = self.get_bucket_collection_restore_cluster_host() self.verify_collections_in_restore_cluster_host()
def test_restore_with_auto_create_buckets(self): """ Restore cluster with --auto-create-buckets option """ self.active_resident_threshold = 100 self.load_all_buckets(self.backupset.cluster_host) self.log.info("*** done to load items to all buckets") self.ops_type = self.input.param("ops-type", "update") self.expected_error = self.input.param("expected_error", None) if self.create_scopes and not self.buckets_only: self.create_scope_cluster_host() if self.create_collections and not self.buckets_only and not self.scopes_only: self.create_collection_cluster_host(self.backupset.col_per_scope) backup_scopes = self.get_bucket_scope_cluster_host() scopes_id = [] for scope in backup_scopes: if scope == "_default": continue scopes_id.append(self.get_scopes_id_cluster_host(scope)) """ remove null and empty element """ scopes_id = [i for i in scopes_id if i] if isinstance(backup_scopes, tuple): backup_scopes = backup_scopes[0] for backup_scope in backup_scopes: backup_collections = self.get_bucket_collection_cluster_host(scope=backup_scope) if isinstance(backup_collections, tuple): backup_collections = backup_collections[0] col_stats = self.get_collection_stats_cluster_host() for backup_scope in backup_scopes: bk_scope_id = self.get_scopes_id_cluster_host(backup_scope) self.backup_create_validate() for i in range(1, self.backupset.number_of_backups + 1): if self.ops_type == "update": self.log.info("*** start to update items in all buckets") col_cmd = "" if self.backupset.load_to_collection: self.backupset.load_scope_id = choice(scopes_id) col_cmd = " -c {0} ".format(self.backupset.load_scope_id) self.load_all_buckets(self.backupset.cluster_host, ratio=0.1, command_options=col_cmd) self.log.info("*** done update items in all buckets") self.sleep(10) self.log.info("*** start to validate backup cluster") self.backup_cluster_validate() self.targetMaster = True start = randrange(1, self.backupset.number_of_backups + 1) if start == self.backupset.number_of_backups: end = start else: end = randrange(start, self.backupset.number_of_backups + 1) self.log.info("*** start to restore cluster") restored = {"{0}/{1}".format(start, end): ""} for i in range(1, self.backupset.number_of_backups + 1): if self.reset_restore_cluster: self.log.info("*** start to reset cluster") self.backup_reset_clusters(self.cluster_to_restore) if self.same_cluster: self._initialize_nodes(Cluster(), self.servers[:self.nodes_init]) else: shell = RemoteMachineShellConnection(self.backupset.restore_cluster_host) shell.enable_diag_eval_on_non_local_hosts() shell.disconnect() rest = RestConnection(self.backupset.restore_cluster_host) rest.force_eject_node() rest.init_node() self.log.info("Done reset cluster") self.sleep(10) """ Add built-in user cbadminbucket to second cluster """ self.add_built_in_server_user(node=self.input.clusters[0][:self.nodes_init][0]) rest = RestConnection(self.backupset.restore_cluster_host) rest.set_indexer_storage_mode(username='******', password='******', storageMode="memory_optimized") self.backupset.start = start self.backupset.end = end self.log.info("*** start restore validation") data_map_collection = [] for scope in backup_scopes: if "default" in scope or scope == '': continue data_map_collection.append(self.buckets[0].name + "." + scope + "=" + \ self.buckets[0].name + "." + scope) self.bucket_map_collection = ",".join(data_map_collection) self.backup_restore_validate(compare_uuid=False, seqno_compare_function=">=", expected_error=self.expected_error)
def test_backup_restore_collection_sanity(self): """ 1. Create default bucket on the cluster and loads it with given number of items 2. Perform updates and create backups for specified number of times (test param number_of_backups) 3. Perform restores for the same number of times with random start and end values """ if self.reset_backup_cluster: self.log.info("*** start to reset backup cluster") self._create_backup_cluster(self.backupset.backup_services_init) self.sleep(10) rest_conn = RestConnection(self.backupset.cluster_host) rest_conn.create_bucket(bucket="default", ramQuotaMB="256") self.buckets = RestConnection(self.master).get_buckets() self.log.info("*** create collection in all buckets") self.log.info("*** start to load items to all buckets") self.active_resident_threshold = 100 self.load_all_buckets(self.backupset.cluster_host) self.log.info("*** done to load items to all buckets") self.ops_type = self.input.param("ops-type", "update") self.expected_error = self.input.param("expected_error", None) if self.create_gsi: self.create_indexes() if self.create_scopes and not self.buckets_only: self.create_scope_cluster_host() if self.create_collections and not self.buckets_only and not self.scopes_only: self.create_collection_cluster_host(self.backupset.col_per_scope) backup_scopes = self.get_bucket_scope_cluster_host() if len(backup_scopes) < 2: self.sleep(4) backup_scopes = self.get_bucket_scope_cluster_host() if backup_scopes[0][:4] == "\x1b[6n": backup_scopes[0] = backup_scopes[0][4:] self.log.info("scopes in backup cluster: {0}".format(backup_scopes)) scopes_id = [] for scope in backup_scopes: if scope == "_default": continue self.log.info("get scope id of scope: {0}".format(scope)) scopes_id.append(self.get_scopes_id_cluster_host(scope)) self.log.info("scope id in backup cluster: {0}".format(scopes_id)) """ remove null and empty element """ scopes_id = [i for i in scopes_id if i] col_stats = self.get_collection_stats_cluster_host() for backup_scope in backup_scopes: bk_scope_id = self.get_scopes_id_cluster_host(backup_scope) if self.auto_failover: self.log.info("Enabling auto failover on " + str(self.backupset.cluster_host)) rest_conn = RestConnection(self.backupset.cluster_host) rest_conn.update_autofailover_settings(self.auto_failover, self.auto_failover_timeout) if self.drop_scopes: self.delete_scope_cluster_host() else: self.delete_collection_cluster_host() rest_bk = RestConnection(self.backupset.cluster_host) cluster_srv = list(rest_bk.get_nodes_services().values()) fts_srv = False index_srv = False for node_srv in cluster_srv: if "fts" in node_srv: fts_srv = True if "index" in node_srv: index_srv = True if index_srv: bk_storage_mode = rest_bk.get_index_settings()["indexer.settings.storage_mode"] else: bk_storage_mode = "plasma" self.backup_create_validate() for i in range(1, self.backupset.number_of_backups + 1): if self.ops_type == "update": self.log.info("*** start to update items in all buckets") col_cmd = "" if self.backupset.load_to_collection: if len(scopes_id) > 1: self.backupset.load_scope_id = choice(scopes_id) else: if scopes_id: self.backupset.load_scope_id = scopes_id[0] else: self.log.info("scopes Id: {0}. Let get scopes again.".format(scopes_id)) bk_scopes = self.get_bucket_scope_cluster_host() for scope in bk_scopes: if scope == "_default" or not scope: continue self.log.info("get scope id of scope: {0}".format(scope)) scopes_id.append(self.get_scopes_id_cluster_host(scope)) self.backupset.load_scope_id = scopes_id[0] col_cmd = " -c {0} ".format(self.backupset.load_scope_id) self.sleep(10, "wait for scopes and collections created") self.load_all_buckets(self.backupset.cluster_host, ratio=0.1, command_options=col_cmd) self.log.info("*** done update items in all buckets") self.sleep(10) self.log.info("*** start to validate backup cluster") self.backup_cluster_validate() self.targetMaster = True start = randrange(1, self.backupset.number_of_backups + 1) if start == self.backupset.number_of_backups: end = start else: end = randrange(start, self.backupset.number_of_backups + 1) self.log.info("*** start to restore cluster") restored = {"{0}/{1}".format(start, end): ""} for i in range(1, self.backupset.number_of_backups + 1): if self.reset_restore_cluster: self.log.info("*** start to reset cluster") self.backup_reset_clusters(self.cluster_to_restore) if self.same_cluster: self._initialize_nodes(Cluster(), self.servers[:self.nodes_init]) else: shell = RemoteMachineShellConnection(self.backupset.restore_cluster_host) shell.enable_diag_eval_on_non_local_hosts() shell.disconnect() rest = RestConnection(self.backupset.restore_cluster_host) rest.force_eject_node() rest.init_node() rest.set_indexer_storage_mode(username='******', password='******', storageMode=bk_storage_mode) self.log.info("Done reset cluster") self.sleep(10) """ Add built-in user cbadminbucket to second cluster """ self.add_built_in_server_user(node=self.input.clusters[0][:self.nodes_init][0]) self.backupset.start = start self.backupset.end = end self.log.info("*** start restore validation") data_map_collection = [] for scope in backup_scopes: if "default" in scope or scope == '': continue data_map_collection.append(self.buckets[0].name + "." + scope + "=" + \ self.buckets[0].name + "." + scope) self.bucket_map_collection = ",".join(data_map_collection) self.backup_restore_validate(compare_uuid=False, seqno_compare_function=">=", expected_error=self.expected_error) if self.backupset.number_of_backups == 1: continue while "{0}/{1}".format(start, end) in restored: start = randrange(1, self.backupset.number_of_backups + 1) if start == self.backupset.number_of_backups: end = start else: end = randrange(start, self.backupset.number_of_backups + 1) restored["{0}/{1}".format(start, end)] = "" if not self.drop_scopes: restore_scopes = self.get_bucket_scope_restore_cluster_host() if not self.drop_collections: self.verify_collections_in_restore_cluster_host() else: try: for scope in restore_scopes: restore_collections = self.get_bucket_collection_restore_cluster_host(scope=scope) if restore_collections: self.fail("Restore should not restore delete collection") except Exception as e: if e: print("Exception: ", str(e)) else: try: for scope in backup_scopes: restore_scopes = self.get_bucket_scope_restore_cluster_host(scope=scope) if restore_scopes: self.fail("Restore should not restore delete scopes") except Exception as e: if e: print("Exception: ", str(e))
def setUp(self): super(ConfigPurging, self).setUp() is_windows = False for node in self.cluster.servers: shell = RemoteMachineShellConnection(node) if shell.info.type.lower() == OS.WINDOWS: is_windows = True shell.enable_diag_eval_on_non_local_hosts() shell.disconnect() self.cluster_util.update_cluster_nodes_service_list(self.cluster) # Default purger values self.default_run_interval = 60 self.default_purge_age = 300 self.time_stamp = time() self.num_index = self.input.param("num_index", 0) self.index_type = self.input.param("index_type", CbServer.Services.FTS) self.index_name_len = self.input.param("index_name_len", 10) self.fts_index_partition = self.input.param("fts_index_partition", 1) self.index_replicas = self.input.param("gsi_index_replicas", 1) self.fts_helper = FtsHelper(self.cluster.fts_nodes[0]) \ if self.cluster.fts_nodes else None self.n1ql_helper = N1QLHelper(server=self.cluster.query_nodes[0], use_rest=True, log=self.log) \ if self.cluster.query_nodes else None self.spare_node = self.servers[-1] self.couchbase_base_dir = "/opt/couchbase" if is_windows: self.couchbase_base_dir = \ "/cygdrive/c/Program\\ Files/Couchbase/Server" # Param order: # fts_name, bucket_name, index_partitions, scope_name, collection_name self.fts_param_template = '{ \ "type": "fulltext-index", \ "name": "%s", \ "sourceType": "gocbcore", \ "sourceName": "%s", \ "sourceUUID": "%s", \ "planParams": { \ "maxPartitionsPerPIndex": 1024, \ "indexPartitions": %d \ }, \ "params": { \ "doc_config": { \ "docid_prefix_delim": "", \ "docid_regexp": "", \ "mode": "scope.collection.type_field", \ "type_field": "type" \ }, \ "mapping": { \ "analysis": {}, \ "default_analyzer": "standard", \ "default_datetime_parser": "dateTimeOptional", \ "default_field": "_all", \ "default_mapping": { \ "dynamic": true, \ "enabled": false \ }, \ "default_type": "_default", \ "docvalues_dynamic": false, \ "index_dynamic": true, \ "store_dynamic": false, \ "type_field": "_type", \ "types": { \ "%s.%s": { \ "dynamic": true, \ "enabled": true \ } \ } \ }, \ "store": { \ "indexType": "scorch", \ "segmentVersion": 15 \ } \ }, \ "sourceParams": {} \ }' self.gsi_index_name_template = "%s_%s_%s_%d" self.gsi_create_template = "CREATE PRIMARY INDEX `%s` " \ "ON `%s`.`%s`.`%s` USING GSI " \ "WITH {\"num_replica\": %d}" self.gsi_drop_template = "DROP INDEX `%s`.`%s` USING GSI" self.op_create = "key_create" self.op_remove = "key_delete" self.ts_during_start = self.__get_current_timestamps_from_debug_log() self.initial_tombstones = \ self.cluster_util.get_metakv_dicts(self.cluster.master) self.log.info(self.ts_during_start)