def load_docs_in_cb_bucket_before_and_after_cbas_connect(self): self.setup_for_test() # Load more docs in Couchbase bucket. self.perform_doc_ops_in_all_cb_buckets("create", self.num_items, self.num_items * 2) self.bucket_util.verify_stats_all_buckets(self.num_items * 2) if self.test_abort_snapshot: self.log.info("Creating sync_write aborts after dataset connect") for server in self.cluster_util.get_kv_nodes(): ssh_shell = RemoteMachineShellConnection(server) cbstats = Cbstats(ssh_shell) replica_vbs = cbstats.vbucket_list( self.bucket_util.buckets[0].name, "replica") load_gen = doc_generator("test_abort_key", self.num_items, self.num_items, target_vbucket=replica_vbs) success = self.bucket_util.load_durable_aborts( ssh_shell, [load_gen], self.bucket_util.buckets[0], self.durability_level, "update", "all_aborts") if not success: self.log_failure("Simulating aborts failed") ssh_shell.disconnect() self.validate_test_failure() # Validate no. of items in CBAS dataset if not self.cbas_util.validate_cbas_dataset_items_count( self.cbas_dataset_name, self.num_items * 2): self.fail("No. of items in CBAS dataset do not match " "that in the CB bucket")
def test_crash_while_streaming(self): bucket = self.bucket_util.buckets[0] vbucket = randint(0, self.vbuckets) nodeA = self.servers[0] self.load_docs(bucket, vbucket, 0, self.num_items, "create") shell_conn = RemoteMachineShellConnection(nodeA) cb_stat_obj = Cbstats(shell_conn) dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER) _ = dcp_client.stream_req(vbucket, 0, 0, 2 * self.num_items, 0) self.load_docs(nodeA, vbucket, self.num_items) self.assertTrue(self.stop_node(0), msg="Failed during stop_node") self.sleep(2, "Sleep after stop_node") self.assertTrue(self.start_node(0), msg="Failed during start_node") rest = RestHelper(RestConnection(nodeA)) self.assertTrue(rest.is_ns_server_running(), msg="Failed while is_ns_server_running check") self.sleep(30, "Sleep to wait for ns_server to run") vb_info = cb_stat_obj.vbucket_seqno(bucket.name) dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, vb_info[vbucket]["high_seqno"], 0) stream.run() self.assertTrue(stream.last_by_seqno == vb_info[vbucket]["high_seqno"], msg="Mismatch in high_seqno. {0} == {1}".format( vb_info[vbucket]["high_seqno"], stream.last_by_seqno)) # Disconnect shell Connection for the node shell_conn.disconnect()
def setUp(self): super(OpsChangeCasTests, self).setUp() self.key = "test_cas" self.expire_time = self.input.param("expire_time", 35) self.item_flag = self.input.param("item_flag", 0) self.load_gen = doc_generator(self.key, 0, self.num_items, doc_size=self.doc_size) self.node_data = dict() for node in self.cluster_util.get_kv_nodes(): shell = RemoteMachineShellConnection(node) cb_stat = Cbstats(shell) self.node_data[node.ip] = dict() self.node_data[node.ip]["shell"] = shell self.node_data[node.ip]["cb_stat"] = Cbstats(shell) self.node_data[node.ip]["active"] = cb_stat.vbucket_list( self.bucket, "active") self.node_data[node.ip]["replica"] = cb_stat.vbucket_list( self.bucket, "replica") if self.sdk_client_pool: self.client = self.sdk_client_pool.get_client_for_bucket( self.bucket) else: self.client = SDKClient([self.cluster.master], self.bucket)
def collect_vbucket_num_stats(self, servers, buckets): """ Method to extract the failovers stats given by cbstats tool Paramters: buckets: bucket informaiton servers: server information Returns: Failover stats as follows: if not collecting per node :: {bucket : [{key:value}]} if collecting per node :: {bucket : {node:[{key:value}]}} """ active_bucketMap = {} replica_bucketMap = {} for bucket in buckets: active_map_data = {} replica_map_data = {} for server in servers: #client = MemcachedClientHelper.direct_client(server, bucket) #stats = client.stats('') cbstat = Cbstats(server) stats = cbstat.vbucket_list(bucket.name) active_map_data[server.ip] = len(stats) stats = cbstat.vbucket_list(bucket.name, vbucket_type="replica") replica_map_data[server.ip] = len(stats) # for key in stats.keys(): # if key == 'vb_active_num': # active_map_data[server.ip] = int(stats[key]) # if key == 'vb_replica_num': # replica_map_data[server.ip] = int(stats[key]) active_bucketMap[bucket.name] = active_map_data replica_bucketMap[bucket.name] = replica_map_data return active_bucketMap, replica_bucketMap
def test_mutations_during_rebalance(self): # start rebalance task = self.cluster.async_rebalance([self.master], self.servers[1:], []) # load some data vbucket = 0 self.load_docs(self.master, vbucket, self.num_items) shell_conn = RemoteMachineShellConnection(self.cluster.master) cb_stat_obj = Cbstats(self.log, shell_conn) # Fetch vbucket seqno stats vb_stat = cb_stat_obj.vbucket_seqno(self.bucket_util.buckets[0].name) # stream self.log.info("Streaming vb {0} to seqno {1}" .format(vbucket, vb_stat[vbucket]["high_seqno"])) self.assertEquals(vb_stat[vbucket]["high_seqno"], self.num_items) dcp_client = self.dcp_client(self.master, PRODUCER, vbucket) stream = dcp_client.stream_req(vbucket, 0, 0, vb_stat[vbucket]["high_seqno"], vb_stat[vbucket]["uuid"]) stream.run() last_seqno = stream.last_by_seqno assert last_seqno == vb_stat[vbucket]["high_seqno"], last_seqno # verify rebalance assert task.result()
def test_create_delete_collection_same_order(self): # Create collection in increasing order shell_conn = RemoteMachineShellConnection(self.cluster.master) cb_stat = Cbstats(shell_conn) collection_count = 1 while collection_count < 1000: doc_loading_spec = \ self.bucket_util.get_crud_template_from_package( "def_add_collection") self.bucket_util.run_scenario_from_spec(self.task, self.cluster, self.bucket_util.buckets, doc_loading_spec, mutation_num=0, batch_size=self.batch_size) collection_count = cb_stat.get_collections(self.bucket)["count"] self.bucket_util.validate_docs_per_collections_all_buckets() # Delete collections while collection_count > 1: doc_loading_spec = \ self.bucket_util.get_crud_template_from_package( "def_drop_collection") self.bucket_util.run_scenario_from_spec(self.task, self.cluster, self.bucket_util.buckets, doc_loading_spec, mutation_num=0, batch_size=self.batch_size) collection_count = cb_stat.get_collections(self.bucket)["count"] # Validate doc count as per bucket collections self.bucket_util.validate_docs_per_collections_all_buckets() self.validate_test_failure()
def get_vbucket_type_mapping(self, bucket_name): for node in self.vbs_in_node.keys(): cb_stat = Cbstats(self.vbs_in_node[node]["shell"]) self.vbs_in_node[node]["active"] = \ cb_stat.vbucket_list(bucket_name, "active") self.vbs_in_node[node]["replica"] = \ cb_stat.vbucket_list(bucket_name, "replica")
def rebalance_out_with_warming_up(self): master_restart = self.input.param("master_restart", False) if master_restart: warmup_node = self.cluster.master else: warmup_node = self.cluster.servers[len(self.cluster.servers) - self.nodes_out - 1] servs_out = self.cluster.servers[len(self.cluster.servers) - self.nodes_out:] if self.test_abort_snapshot: self.log.info("Creating sync_write abort scenario for replica vbs") for server in self.cluster_util.get_kv_nodes(self.cluster): ssh_shell = RemoteMachineShellConnection(server) cbstats = Cbstats(ssh_shell) replica_vbs = cbstats.vbucket_list( self.cluster.buckets[0].name, "replica") load_gen = doc_generator(self.key, 0, 5000, target_vbucket=replica_vbs) success = self.bucket_util.load_durable_aborts( ssh_shell, [load_gen], self.cluster.buckets[0], self.durability_level, "update", "all_aborts") if not success: self.log_failure("Simulating aborts failed") ssh_shell.disconnect() self.validate_test_failure() shell = RemoteMachineShellConnection(warmup_node) shell.stop_couchbase() self.sleep(20) shell.start_couchbase() shell.disconnect() # Workaround for Eph case (MB-44682 - Not a bug) if self.bucket_type == Bucket.Type.EPHEMERAL: self.sleep(15, "Wait for couchbase server to start") rebalance = self.task.async_rebalance( self.cluster.servers, [], servs_out) self.task.jython_task_manager.get_task_result(rebalance) self.assertTrue(rebalance.result, "Rebalance Failed") self.cluster.nodes_in_cluster = list(set(self.cluster.nodes_in_cluster) - set(servs_out)) if rebalance.result is False: self.log.info("Rebalance was failed as expected") self.assertTrue(self.bucket_util._wait_warmup_completed( self.cluster_util.get_kv_nodes(self.cluster), self.cluster.buckets[0], wait_time=self.wait_timeout * 10)) self.log.info("Second attempt to rebalance") rebalance = self.task.async_rebalance( self.cluster.servers, [], servs_out) self.task.jython_task_manager.get_task_result(rebalance) self.assertTrue(rebalance.result, "Rebalance attempt failed again") self.cluster.nodes_in_cluster = list(set(self.cluster.nodes_in_cluster) - set(servs_out)) if not self.atomicity: self.bucket_util.verify_cluster_stats(self.cluster, self.num_items, timeout=self.wait_timeout) self.bucket_util.verify_unacked_bytes_all_buckets(self.cluster)
def setUp(self): super(OutOfOrderReturns, self).setUp() self.ooo_order = 0 self.test_lock = Lock() self.doc_ops = self.input.param("doc_ops", "update;update").split(";") # Initialize cluster using given nodes nodes_init = self.cluster.servers[1:self.nodes_init] \ if self.nodes_init != 1 else [] self.task.rebalance([self.cluster.master], nodes_init, []) self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init) # Disable auto-failover to avoid failover of nodes status = RestConnection(self.cluster.master) \ .update_autofailover_settings(False, 120, False) self.assertTrue(status, msg="Failure during disabling auto-failover") # Create default bucket and add rbac user self.bucket_util.create_default_bucket( bucket_type=self.bucket_type, storage=self.bucket_storage, ram_quota=self.bucket_size, replica=self.num_replicas, compression_mode=self.compression_mode, eviction_policy=self.bucket_eviction_policy) self.cluster.nodes_in_cluster.extend([self.cluster.master]) self.bucket = self.bucket_util.buckets[0] # Create sdk_clients for pool if self.sdk_client_pool: self.log.info("Creating SDK client pool") self.sdk_client_pool.create_clients( self.bucket, self.cluster.nodes_in_cluster, req_clients=self.sdk_pool_capacity, compression_settings=self.sdk_compression) # Create shell connection to each kv_node for cbstat object self.kv_nodes = self.cluster_util.get_kv_nodes() self.node_data = dict() for node in self.kv_nodes: shell = RemoteMachineShellConnection(node) cb_stat = Cbstats(shell) self.node_data[node] = dict() self.node_data[node]["shell"] = shell self.node_data[node]["cb_stat"] = cb_stat self.node_data[node]["active_vbs"] = \ cb_stat.vbucket_list(self.bucket.name, vbucket_type="active") self.node_data[node]["replica_vbs"] = \ cb_stat.vbucket_list(self.bucket.name, vbucket_type="replica") # Print cluster & bucket stats self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats()
def test_failover_swap_rebalance(self): """ add and failover node then perform swap rebalance """ assert len(self.servers) > 2, "not enough servers" nodeA = self.servers[0] nodeB = self.servers[1] nodeC = self.servers[2] gen_create = doc_generator('dcp', 0, self.num_items, doc_size=64) self._load_all_buckets(nodeA, gen_create, "create", 0) vbucket = 0 # rebalance in nodeB assert self.cluster.rebalance([nodeA], [nodeB], []) # add nodeC rest = RestConnection(nodeB) rest.add_node(user=nodeC.rest_username, password=nodeC.rest_password, remoteIp=nodeC.ip, port=nodeC.port) # stop and failover nodeA assert self.stop_node(0) self.stopped_nodes.append(0) self.master = nodeB assert self.cluster.failover([nodeB], [nodeA]) try: assert self.cluster.rebalance([nodeB], [], []) except Exception: pass self.add_built_in_server_user() # verify seqnos and stream mutations rest = RestConnection(nodeB) total_mutations = 0 # Create connection for CbStats shell_conn = RemoteMachineShellConnection(self.cluster.master) cb_stat_obj = Cbstats(shell_conn) vb_info = cb_stat_obj.vbucket_seqno(self.bucket_util.buckets[0].name) for vb in range(0, self.vbuckets): total_mutations += int(vb_info[vb]["high_seqno"]) # Disconnect the Cbstats shell_conn shell_conn.disconnect() # / 2 # divide by because the items are split between 2 servers self.assertTrue(total_mutations == self.num_items, msg="Number mismatch. {0} != {1}".format( total_mutations, self.num_items)) task = self.cluster.async_rebalance([nodeB], [], [nodeC]) task.result()
def get_collection_id(self, bucket_name, scope_name, collection_name=None): shell = RemoteMachineShellConnection(self.cluster.master) cbstats = Cbstats(shell) if collection_name: field = scope_name + ':' + collection_name + ':' + 'id:' cid_stat = cbstats.get_stats(bucket_name, "collections", field)[0][0] else: field = scope_name + ':id:' cid_stat = cbstats.get_stats(bucket_name, "scopes", field)[0][0] return cid_stat.split('0x')[-1].strip()
def get_magma_stats(self, bucket, servers=None): magma_stats_for_all_servers = dict() servers = servers or self.cluster.nodes_in_cluster if type(servers) is not list: servers = [servers] for server in servers: result = dict() cbstat_obj = Cbstats(server) result = cbstat_obj.magma_stats(bucket.name) magma_stats_for_all_servers[server.ip] = result return magma_stats_for_all_servers
def verify_vbucket_details_stats(self, bucket, kv_servers, vbuckets=1024, expected_val=dict()): """ :param bucket: Bucket object :param kv_servers: List of kv_nodes currently present in the cluster :param vbuckets: Total vbucket count for the bucket. Default 1024 :param expected_val: dict() containing expected key,value pairs :return verification_failed: Bool value denoting verification failed or not """ verification_failed = False vb_details_stats = dict() ops_val = dict() ops_val["ops_create"] = 0 ops_val["ops_delete"] = 0 ops_val["ops_update"] = 0 ops_val["ops_reject"] = 0 ops_val["ops_get"] = 0 ops_val["rollback_item_count"] = 0 ops_val["sync_write_aborted_count"] = 0 ops_val["sync_write_committed_count"] = 0 ops_val["pending_writes"] = 0 # Fetch stats for all available vbuckets into 'vb_details_stats' for server in kv_servers: shell = RemoteMachineShellConnection(server) cbstat_obj = Cbstats(shell) vb_details_stats.update(cbstat_obj.vbucket_details(bucket.name)) shell.disconnect() for vb_num in range(0, vbuckets): vb_num = str(vb_num) for op_type in ops_val.keys(): ops_val[op_type] += int(vb_details_stats[vb_num][op_type]) # Verification block for op_type in ops_val.keys(): self.log.debug("%s for %s: %s" % (op_type, bucket.name, ops_val[op_type])) if op_type in expected_val \ and not DurabilityHelper.__compare(ops_val[op_type], expected_val[op_type], DurabilityHelper.EQUAL): verification_failed = True self.log.error( "Mismatch in %s stats. %s != %s" % (op_type, ops_val[op_type], expected_val[op_type])) return verification_failed
def test_failover_expired_items_in_vB(self): self.maxttl = 120 self.doc_ops = "expiry" self.expiry_perc = self.input.param("expiry_perc", 100) shell_conn = RemoteMachineShellConnection( self.cluster.nodes_in_cluster[-1]) cbstats = Cbstats(shell_conn) self.target_vbucket = cbstats.vbucket_list( self.bucket_util.buckets[0].name) self.generate_docs(target_vbucket=self.target_vbucket) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep( self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") self.sleep( self.exp_pager_stime * 10, "Wait for KV purger to scan expired docs and add \ tombstones.") self.task.async_failover(self.cluster.nodes_in_cluster, self.cluster.nodes_in_cluster[-1], graceful=True) self.nodes = self.rest.node_statuses() self.task.rebalance(self.cluster.nodes_in_cluster, to_add=[], to_remove=[self.cluster.nodes_in_cluster[-1]]) # Metadata Purge Interval self.meta_purge_interval = 60 self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) self.sleep( self.meta_purge_interval * 2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") self.log.info("Starting compaction for each bucket") self.run_compaction() # All docs and tomb-stone should be dropped from the storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after full compaction: {}".format(ts))
def get_magma_stats(self, bucket, servers=None, field_to_grep=None): magma_stats_for_all_servers = dict() servers = servers or self.cluster.nodes_in_cluster if type(servers) is not list: servers = [servers] for server in servers: result = dict() shell = RemoteMachineShellConnection(server) cbstat_obj = Cbstats(shell) result = cbstat_obj.magma_stats(bucket.name, field_to_grep=field_to_grep) shell.disconnect() magma_stats_for_all_servers[server.ip] = result return magma_stats_for_all_servers
def collect_compare_dcp_stats(self, buckets, servers, perNode=True, stat_name='unacked_bytes', compare_value=0, flow_control_buffer_size=20971520, filter_list=[]): """ Method to extract the failovers stats given by cbstats tool Paramters: buckets: bucket informaiton servers: server information stat_name: stat we are searching to compare compare_value: the comparison value to be satisfied Returns: map of bucket informing if stat matching was satisfied / not satisfied example:: unacked_bytes in dcp """ bucketMap = dict() for bucket in buckets: bucketMap[bucket.name] = True for bucket in buckets: for server in servers: # client = MemcachedClientHelper.direct_client(server, bucket) # stats = client.stats('dcp') shell = RemoteMachineShellConnection(server) cbstat = Cbstats(shell) stats = cbstat.dcp_stats(bucket) for key in stats.keys(): do_filter = False if stat_name in key: for filter_key in filter_list: if filter_key in key: do_filter = True value = int(stats[key]) if not do_filter: if value != compare_value: if "eq_dcpq:mapreduce_view" in key: if value >= flow_control_buffer_size: bucketMap[bucket] = False else: bucketMap[bucket] = False return bucketMap
def map_collection_data(self): cb_stat_objects = list() collection_data = None for node in self.cluster_util.get_kv_nodes(): shell = RemoteMachineShellConnection(node) cb_stat_objects.append(Cbstats(shell)) for cb_stat in cb_stat_objects: tem_collection_data = cb_stat.get_collections(self.bucket) if collection_data is None: collection_data = tem_collection_data else: for key, value in tem_collection_data.items(): if type(value) is dict: for col_name, c_data in value.items(): collection_data[key][col_name]['items'] \ += c_data['items'] for s_name, s_data in collection_data.items(): if type(s_data) is not dict: continue self.bucket_util.create_scope_object(self.bucket, {"name": s_name}) for c_name, c_data in s_data.items(): if type(c_data) is not dict: continue self.bucket_util.create_collection_object( self.bucket, s_name, {"name": c_name, "num_items": c_data["items"], "maxTTL": c_data.get("maxTTL", 0)}) # Close shell connections for cb_stat in cb_stat_objects: cb_stat.shellConn.disconnect()
def check(self): # check bucket compaction status across all nodes nodes = self.rest.get_nodes() current_compaction_count = {} for node in nodes: current_compaction_count[node.ip] = 0 s = TestInputServer() s.ip = node.ip s.ssh_username = self.server.ssh_username s.ssh_password = self.server.ssh_password shell = RemoteMachineShellConnection(s) res = Cbstats(shell).get_kvtimings() shell.disconnect() for i in res[0]: # check for lines that look like # rw_0:compact_131072,262144: 8 if 'compact' in i: current_compaction_count[node.ip] += int(i.split(':')[2]) if cmp(current_compaction_count, self.compaction_count) == 1: # compaction count has increased self.set_result(True) self.state = FINISHED else: if self.retries > 0: # retry self.retries = self.retries - 1 self.task_manager.schedule(self, 10) else: # never detected a compaction task running self.set_result(False) self.state = FINISHED
def setUp(self): super(RollbackTests, self).setUp() self.num_rollbacks = self.input.param("num_rollbacks", 2) self.collection_ops_type = self.input.param("collection_ops", None) self.rollback_with_multiple_mutation = self.input.param( "rollback_with_multiple_mutation", False) self.bucket = self.bucket_util.buckets[0] # Disable auto-fail_over to avoid fail_over of nodes status = RestConnection(self.cluster.master) \ .update_autofailover_settings(False, 120, False) self.assertTrue(status, msg="Failure during disabling auto-failover") # Used to calculate expected queue size of validation before rollback self.total_rollback_items = 0 self.kv_nodes = self.cluster_util.get_kv_nodes() self.sync_write_enabled = self.durability_helper.is_sync_write_enabled( self.bucket_durability_level, self.durability_level) # Open shell connections to kv nodes and create cbstat objects self.node_shells = dict() for node in self.kv_nodes: shell_conn = RemoteMachineShellConnection(node) self.node_shells[node] = dict() self.node_shells[node]["shell"] = shell_conn self.node_shells[node]["cbstat"] = Cbstats(shell_conn)
def ops_change_cas(self): """ CAS value manipulation by update, delete, expire test. We load a certain number of items. Then for half of them, we use MemcachedClient cas() method to mutate those item values in order to change CAS value of those items. We use MemcachedClient set() to set a quarter of the items expired. We also use MemcachedClient delete() to delete a quarter of the items """ gen_load = doc_generator('nosql', 0, self.num_items, doc_size=self.doc_size) gen_update = doc_generator('nosql', 0, self.num_items / 2, doc_size=self.doc_size) gen_delete = doc_generator('nosql', self.num_items / 2, (self.num_items * 3 / 4), doc_size=self.doc_size) gen_expire = doc_generator('nosql', (self.num_items * 3 / 4), self.num_items, doc_size=self.doc_size) self._load_all_buckets(gen_load, "create") self.bucket_util.verify_stats_all_buckets(self.cluster, self.num_items) self.bucket_util._wait_for_stats_all_buckets(self.cluster, self.cluster.buckets) # Create cbstat objects self.shell_conn = dict() self.cb_stat = dict() self.vb_details = dict() for node in self.cluster_util.get_kv_nodes(self.cluster): self.vb_details[node.ip] = dict() self.vb_details[node.ip]["active"] = list() self.vb_details[node.ip]["replica"] = list() self.shell_conn[node.ip] = RemoteMachineShellConnection(node) self.cb_stat[node.ip] = Cbstats(self.shell_conn[node.ip]) self.vb_details[node.ip]["active"] = \ self.cb_stat[node.ip].vbucket_list(self.bucket.name, "active") self.vb_details[node.ip]["replica"] = \ self.cb_stat[node.ip].vbucket_list(self.bucket.name, "replica") if self.doc_ops is not None: if "update" in self.doc_ops: self.verify_cas("update", gen_update) if "touch" in self.doc_ops: self.verify_cas("touch", gen_update) if "delete" in self.doc_ops: self.verify_cas("delete", gen_delete) if "expire" in self.doc_ops: self.verify_cas("expire", gen_expire) self.bucket_util._wait_for_stats_all_buckets(self.cluster, self.cluster.buckets) self.validate_test_failure()
def test_stream_all_buckets(self): doc_gen = doc_generator(self.key, 0, self.num_items) self._load_all_buckets(self.master, doc_gen, "create", 0) user_name = self.input.param("user_name", None) password = self.input.param("password", None) nodeA = self.servers[0] vbuckets = [vb for vb in range(self.vbuckets)] shell_conn = RemoteMachineShellConnection(nodeA) cb_stat_obj = Cbstats(shell_conn) for bucket in self.bucket_util.buckets: if user_name is not None: self.add_built_in_server_user([{ 'id': user_name, 'name': user_name, 'password': password }], [{ 'id': user_name, 'name': user_name, 'roles': 'data_dcp_reader[default]' }], self.master) dcp_client = self.dcp_client(nodeA, PRODUCER, bucket_name=bucket.name, auth_user=user_name, auth_password=password) else: dcp_client = self.dcp_client(nodeA, PRODUCER, bucket_name=bucket) vb_info = cb_stat_obj.vbucket_seqno(bucket.name) for vb in vbuckets[0:16]: vbucket = vb.id stream = dcp_client.stream_req(vbucket, 0, 0, vb_info[vb]["high_seqno"], vb_info[vb]["uuid"]) _ = stream.run() self.assertTrue( vb_info[vb]["high_seqno"] == stream.last_by_seqno, msg="Mismatch in high_seqno. {0} == {1}".format( vb_info[vb]["high_seqno"], stream.last_by_seqno)) # Disconnect the shell_conn shell_conn.disconnect()
def tearDown(self): shell = RemoteMachineShellConnection(self.cluster.master) cbstat_obj = Cbstats(shell) for bucket in self.bucket_util.buckets: result = cbstat_obj.all_stats(bucket.name) self.log.info("Bucket: %s, Active Resident ratio(DGM): %s%%" % (bucket.name, result["vb_active_perc_mem_resident"])) self.log.info("Bucket: %s, Replica Resident ratio(DGM): %s%%" % (bucket.name, result["vb_replica_perc_mem_resident"])) if not self.skip_collections_cleanup: self.bucket_util.remove_scope_collections_for_bucket(bucket) shell.disconnect() if self.validate_docs_count_during_teardown: self.bucket_util.validate_docs_per_collections_all_buckets() super(CollectionBase, self).tearDown()
def get_vbucket_info_from_failover_nodes(self): """ Fetch active/replica vbucket list from the nodes which are going to be failed over """ bucket = self.bucket_util.buckets[0] # Reset the values self.active_vb_in_failover_nodes = list() self.replica_vb_in_failover_nodes = list() # Fetch new vbucket list for node in self.server_to_fail: shell_conn = RemoteMachineShellConnection(node) cbstat = Cbstats(shell_conn) self.active_vb_in_failover_nodes += cbstat.vbucket_list( bucket.name, "active") self.replica_vb_in_failover_nodes += cbstat.vbucket_list( bucket.name, "replica")
def setUp(self): super(OutOfOrderReturns, self).setUp() # Create default bucket self.create_bucket(self.cluster) self.ooo_order = 0 self.test_lock = Lock() self.doc_ops = self.input.param("doc_ops", "update;update").split(";") # Disable auto-failover to avoid failover of nodes status = RestConnection(self.cluster.master) \ .update_autofailover_settings(False, 120, False) self.assertTrue(status, msg="Failure during disabling auto-failover") self.cluster.nodes_in_cluster.extend([self.cluster.master]) self.bucket = self.cluster.buckets[0] # Create sdk_clients for pool if self.sdk_client_pool: self.log.info("Creating SDK client pool") self.sdk_client_pool.create_clients( self.bucket, self.cluster.nodes_in_cluster, req_clients=self.sdk_pool_capacity, compression_settings=self.sdk_compression) # Create shell connection to each kv_node for cbstat object self.kv_nodes = self.cluster_util.get_kv_nodes(self.cluster) self.node_data = dict() for node in self.kv_nodes: shell = RemoteMachineShellConnection(node) cb_stat = Cbstats(shell) self.node_data[node] = dict() self.node_data[node]["shell"] = shell self.node_data[node]["cb_stat"] = cb_stat self.node_data[node]["active_vbs"] = \ cb_stat.vbucket_list(self.bucket.name, vbucket_type="active") self.node_data[node]["replica_vbs"] = \ cb_stat.vbucket_list(self.bucket.name, vbucket_type="replica") # Print cluster & bucket stats self.cluster_util.print_cluster_stats(self.cluster) self.bucket_util.print_bucket_stats(self.cluster)
def test_stream_after_n_crashes(self): crashes = self.input.param("crash_num", 5) vbucket = randint(0, self.vbuckets) bucket = self.bucket_util.buckets[0] self.log.info("Chosen vbucket {0} for {1} crashes".format( vbucket, crashes)) start = 0 end = self.num_items nodeA = self.cluster.servers[0] shell_conn = RemoteMachineShellConnection(nodeA) cb_stat_obj = Cbstats(shell_conn) rest = RestHelper(RestConnection(nodeA)) for _ in xrange(crashes): # Load data into the selected vbucket self.load_docs(bucket, vbucket, start, end, "create") self.assertTrue(self.stop_node(0), msg="Failed during stop_node") self.sleep(5, "Sleep after stop_node") self.assertTrue(self.start_node(0), msg="Failed during start_node") self.assertTrue(rest.is_ns_server_running(), msg="Failed while is_ns_server_running check") self.sleep(5, "Waiting after ns_server started") # Fetch vbucket seqno stats vb_stat = cb_stat_obj.vbucket_seqno(bucket.name) dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, vb_stat[vbucket]["high_seqno"], vb_stat[vbucket]["uuid"]) stream.run() self.assertTrue( stream.last_by_seqno == vb_stat[vbucket]["high_seqno"], msg="Mismatch in high_seqno. {0} == {1}".format( vb_stat[vbucket]["high_seqno"], stream.last_by_seqno)) # Update start/end values for next loop start = end end += self.num_items # Disconnect shell Connection for the node shell_conn.disconnect()
def test_crash_entire_cluster(self): self.cluster.rebalance([self.master], self.servers[1:], []) bucket = self.bucket_util.buckets[0] vbucket = randint(0, self.vbuckets) nodeA = self.servers[0] self.load_docs(bucket, vbucket, 0, self.num_items, "create") shell_conn = RemoteMachineShellConnection(nodeA) cb_stat_obj = Cbstats(shell_conn) dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER) _ = dcp_client.stream_req(vbucket, 0, 0, 2 * self.num_items, 0) self.load_docs(nodeA, vbucket, self.num_items) # stop all nodes node_range = range(len(self.servers)) for i in node_range: self.assertTrue(self.stop_node(i), msg="Failed during stoip_node") self.sleep(2, "Wait after stop_node") # start all nodes in reverse order node_range.reverse() for i in node_range: self.assertTrue(self.start_node(i), msg="Failed during start_node") rest = RestHelper(RestConnection(nodeA)) self.assertTrue(rest.is_ns_server_running(), msg="Failed while is_ns_server_running check") vb_info = cb_stat_obj.vbucket_seqno(bucket.name) dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, vb_info[vbucket]["high_seqno"], 0) stream.run() self.assertTrue(stream.last_by_seqno == vb_info[vbucket]["high_seqno"], msg="Seq-no mismatch. {0} != {1}".format( stream.last_by_seqno, vb_info[vbucket]["high_seqno"])) # Disconnect shell Connection for the node shell_conn.disconnect()
def test_buffer_ack_during_dcp_commit(self): """ MB-46482 - Create bucket with min_ram - Perform huge number of sync_writes - Validate 'dcp unacked_bytes' stats are all ZERO """ if self.durability_level == "" \ or self.durability_level.upper() == "NONE": self.fail("Test requires valid durability level for sync_writes") doc_gen = doc_generator(self.key, self.num_items, self.num_items * 3, key_size=10, doc_size=5) self.log.info("Loading %s keys into the bucket" % (self.num_items * 2)) load_task = self.task.async_load_gen_docs( self.cluster, self.bucket, doc_gen, DocLoading.Bucket.DocOps.UPDATE, durability=self.durability_level, print_ops_rate=False) self.task_manager.get_task_result(load_task) self.bucket_util._wait_for_stats_all_buckets(self.cluster, self.cluster.buckets) self.sleep(5, "Wait for dcp") for node in self.cluster_util.get_kv_nodes(self.cluster): cb_stat = Cbstats(node) dcp_stats = cb_stat.dcp_stats(self.bucket.name) for stat_name, val in dcp_stats.items(): if stat_name.split(":")[-1] == "unacked_bytes": self.log.debug("%s: %s" % (stat_name, val)) if int(val) != 0: self.log_failure("%s: %s != 0" % (stat_name, val)) self.validate_test_failure()
def online_swap(self, node_to_upgrade, version, install_on_spare_node=True): vb_details = dict() vb_verification = dict() vb_types = ["active", "replica"] # Fetch active services on node_to_upgrade rest = self.__get_rest_node(node_to_upgrade) services = rest.get_nodes_services() services_on_target_node = services[(node_to_upgrade.ip + ":" + node_to_upgrade.port)] # Record vbuckets in swap_node if CbServer.Services.KV in services_on_target_node: cbstats = Cbstats(node_to_upgrade) for vb_type in vb_types: vb_details[vb_type] = \ cbstats.vbucket_list(self.bucket.name, vb_type) if install_on_spare_node: # Install target version on spare node self.install_version_on_node([self.spare_node], version) # Perform swap rebalance for node_to_upgrade <-> spare_node rebalance_passed = self.task.rebalance( self.cluster_util.get_nodes(self.cluster.master), to_add=[self.spare_node], to_remove=[node_to_upgrade], check_vbucket_shuffling=False, services=[",".join(services_on_target_node)]) if not rebalance_passed: self.log_failure("Swap rebalance failed during upgrade of {0}" .format(node_to_upgrade)) # VBuckets shuffling verification if CbServer.Services.KV in services_on_target_node: # Fetch vbucket stats after swap rebalance for verification cbstats = Cbstats(self.spare_node) for vb_type in vb_types: vb_verification[vb_type] = \ cbstats.vbucket_list(self.bucket.name, vb_type) # Check vbuckets are shuffled or not for vb_type in vb_types: if vb_details[vb_type].sort() \ != vb_verification[vb_type].sort(): self.log_failure("%s vbuckets shuffled post swap_rebalance" % vb_type) self.log.error("%s vbuckets before vs after: %s != %s" % (vb_type, vb_details[vb_type], vb_verification[vb_type])) # Update master node self.cluster.master = self.spare_node self.cluster.nodes_in_cluster.append(self.spare_node) # Update spare_node to rebalanced-out node self.spare_node = node_to_upgrade self.cluster.nodes_in_cluster.remove(node_to_upgrade)
def tearDown(self): cbstat_obj = Cbstats(self.cluster.master) for bucket in self.cluster.buckets: if bucket.bucketType != Bucket.Type.MEMCACHED: result = cbstat_obj.all_stats(bucket.name) self.log.info("Bucket: %s, Active Resident ratio(DGM): %s%%" % (bucket.name, result["vb_active_perc_mem_resident"])) self.log.info("Bucket: %s, Replica Resident ratio(DGM): %s%%" % (bucket.name, result["vb_replica_perc_mem_resident"])) if not self.skip_collections_cleanup \ and bucket.bucketType != Bucket.Type.MEMCACHED: self.bucket_util.remove_scope_collections_for_bucket( self.cluster, bucket) if self.validate_docs_count_during_teardown: self.bucket_util.validate_docs_per_collections_all_buckets( self.cluster) if self.disk_optimized_thread_settings: self.set_num_writer_and_reader_threads(num_writer_threads="default", num_reader_threads="default") super(CollectionBase, self).tearDown()
def test_magma_rollback_to_0(self): items = self.num_items mem_only_items = self.input.param("rollback_items", 10000) if self.nodes_init < 2 or self.num_replicas < 1: self.fail("Not enough nodes/replicas in the cluster/bucket\ to test rollback") self.num_rollbacks = self.input.param("num_rollbacks", 10) shell = RemoteMachineShellConnection(self.cluster_util.cluster.master) self.target_vbucket = Cbstats(shell).vbucket_list( self.bucket_util.buckets[0].name) start = self.num_items # Stopping persistence on NodeA mem_client = MemcachedClientHelper.direct_client( self.input.servers[0], self.bucket_util.buckets[0]) mem_client.stop_persistence() for i in xrange(1, self.num_rollbacks + 1): self.gen_create = doc_generator( self.key, start, mem_only_items, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value) self.loadgen_docs(_sync=True) start = self.gen_create.key_counter stat_map = {self.cluster.nodes_in_cluster[0]: mem_only_items * i} for node in self.cluster.nodes_in_cluster[1:]: stat_map.update({node: 0}) for bucket in self.bucket_util.buckets: self.bucket_util._wait_for_stat(bucket, stat_map) self.sleep(60) shell.kill_memcached() self.assertTrue( self.bucket_util._wait_warmup_completed( [self.cluster_util.cluster.master], self.bucket_util.buckets[0], wait_time=self.wait_timeout * 10)) self.bucket_util.verify_stats_all_buckets(items) shell.disconnect()