def test_doc_key_size(self): """ Insert document key with min and max key size on each available collection and validate :return: """ min_doc_gen = doc_generator("test_min_key_size", 0, self.num_items, key_size=1, doc_size=self.doc_size, mix_key_size=False, randomize_doc_size=False) max_doc_gen = doc_generator("test_max_key_size", 0, self.num_items, key_size=245, doc_size=self.doc_size, mix_key_size=False, randomize_doc_size=False) # Set to keep track of all inserted CAS values known_cas = set() # Client to insert docs under different collections client = SDKClient([self.cluster.master], self.bucket, compression_settings=self.sdk_compression) for doc_gen in [min_doc_gen, max_doc_gen]: while doc_gen.has_next(): key, value = doc_gen.next() for _, scope in self.bucket.scopes.items(): for _, collection in scope.collections.items(): client.select_collection(scope.name, collection.name) result = client.crud("create", key, value, self.maxttl, durability=self.durability_level, timeout=self.sdk_timeout, time_unit="seconds") if result["status"] is False: self.log_failure( "Doc create failed for key '%s' " "collection::scope %s::%s - %s" % (key, scope.name, collection.name, result)) else: if result["cas"] in known_cas: self.log_failure( "Same CAS exists under different " "collection: %s" % result) collection.num_items += 1 known_cas.add(result["cas"]) # Close SDK connection client.close() # Validate doc count as per bucket collections self.bucket_util.validate_docs_per_collections_all_buckets() self.validate_test_failure()
def _dockey_data_ops(self, dockey="dockey"): if self.target_vbucket is None: gen_load = doc_generator(dockey, 0, self.num_items, doc_type="json") else: gen_load = doc_generator(dockey, 0, self.num_items, doc_type="json", target_vbucket=[self.target_vbucket]) bucket = self.bucket_util.get_all_buckets()[0] for op_type in ["create", "update", "delete"]: task = self.task.async_load_gen_docs( self.cluster, bucket, gen_load, op_type, 0, batch_size=20, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries) self.task.jython_task_manager.get_task_result(task) if op_type == "delete": self.num_items = 0 self._persist_and_verify()
def touch_test(self): self.log.info("1. Loading initial set of documents") load_gen = doc_generator(self.key, 0, self.num_items, doc_size=self.doc_size) self._load_all_buckets(load_gen, "create") self.bucket_util.verify_stats_all_buckets(self.num_items) self.bucket_util._wait_for_stats_all_buckets() self.log.info("2. Loading bucket into DGM") dgm_gen = doc_generator( self.key, self.num_items, self.num_items+1) dgm_task = self.task.async_load_gen_docs( self.cluster, self.bucket_util.buckets[0], dgm_gen, "create", 0, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, batch_size=10, process_concurrency=4, active_resident_threshold=self.active_resident_threshold) self.task_manager.get_task_result(dgm_task) self.log.info("3. Touch intial self.num_items docs which are " "residing on disk due to DGM") client = SDKClient([self.cluster.master], self.bucket_util.buckets[0]) while load_gen.has_next(): key, _ = load_gen.next() result = client.crud("touch", key, durability=self.durability_level, timeout=self.sdk_timeout) if result["status"] is not True: self.log_failure("Touch on %s failed: %s" % (key, result)) client.close() self.validate_test_failure()
def ops_change_cas(self): """ CAS value manipulation by update, delete, expire test. We load a certain number of items. Then for half of them, we use MemcachedClient cas() method to mutate those item values in order to change CAS value of those items. We use MemcachedClient set() to set a quarter of the items expired. We also use MemcachedClient delete() to delete a quarter of the items """ gen_load = doc_generator('nosql', 0, self.num_items, doc_size=self.doc_size) gen_update = doc_generator('nosql', 0, self.num_items / 2, doc_size=self.doc_size) gen_delete = doc_generator('nosql', self.num_items / 2, (self.num_items * 3 / 4), doc_size=self.doc_size) gen_expire = doc_generator('nosql', (self.num_items * 3 / 4), self.num_items, doc_size=self.doc_size) self._load_all_buckets(gen_load, "create") self.bucket_util.verify_stats_all_buckets(self.cluster, self.num_items) self.bucket_util._wait_for_stats_all_buckets(self.cluster, self.cluster.buckets) # Create cbstat objects self.shell_conn = dict() self.cb_stat = dict() self.vb_details = dict() for node in self.cluster_util.get_kv_nodes(self.cluster): self.vb_details[node.ip] = dict() self.vb_details[node.ip]["active"] = list() self.vb_details[node.ip]["replica"] = list() self.shell_conn[node.ip] = RemoteMachineShellConnection(node) self.cb_stat[node.ip] = Cbstats(self.shell_conn[node.ip]) self.vb_details[node.ip]["active"] = \ self.cb_stat[node.ip].vbucket_list(self.bucket.name, "active") self.vb_details[node.ip]["replica"] = \ self.cb_stat[node.ip].vbucket_list(self.bucket.name, "replica") if self.doc_ops is not None: if "update" in self.doc_ops: self.verify_cas("update", gen_update) if "touch" in self.doc_ops: self.verify_cas("touch", gen_update) if "delete" in self.doc_ops: self.verify_cas("delete", gen_delete) if "expire" in self.doc_ops: self.verify_cas("expire", gen_expire) self.bucket_util._wait_for_stats_all_buckets(self.cluster, self.cluster.buckets) self.validate_test_failure()
def start_doc_loading_tasks(self, target_vbuckets, scope_name, collection_obj): # Create doc_generator targeting only the active/replica vbuckets # present in the target_node transaction_gen_load = doc_generator( "transaction_key", self.num_items, self.new_docs_to_add, key_size=self.key_size, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=target_vbuckets, vbuckets=self.cluster_util.vbuckets) gen_load = doc_generator(self.key, self.num_items, self.new_docs_to_add, key_size=self.key_size, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=target_vbuckets, vbuckets=self.cluster_util.vbuckets) if self.atomicity: self.transaction_load_task = \ self.task.async_load_gen_docs_atomicity( self.cluster, self.bucket_util.buckets, transaction_gen_load, DocLoading.Bucket.DocOps.CREATE, exp=0, batch_size=10, process_concurrency=self.process_concurrency, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, update_count=self.update_count, transaction_timeout=self.transaction_timeout, commit=True, sync=self.sync) collection_obj.num_items += self.new_docs_to_add elif self.N1qltxn: self.N1ql_load_task = self.task.async_n1qlTxn_query( self.stmts, n1ql_helper=self.n1ql_helper, commit=True, scan_consistency="REQUEST_PLUS") self.doc_loading_task = self.task.async_load_gen_docs( self.cluster, self.bucket, gen_load, DocLoading.Bucket.DocOps.CREATE, exp=0, batch_size=10, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, scope=scope_name, collection=collection_obj.name, skip_read_on_error=True) collection_obj.num_items += self.new_docs_to_add
def test_non_overlapping_parallel_cruds(self): """ Test to run non-overlapping durability cruds on single bucket and make sure all CRUD operation succeeds 1. Run single task_1 with durability operation 2. Create parallel task to run either SyncWrite / Non-SyncWrite operation based on the config param and run that over the docs such that it will not overlap with the other tasks 3. Make sure all CRUDs succeeded without any unexpected exceptions """ doc_ops = self.input.param("doc_ops", "create;delete;update;read") doc_ops = doc_ops.split(";") half_of_num_items = int(self.num_items/2) doc_gen = dict() tasks = list() # Create required doc_generators for CRUD ops doc_gen["create"] = doc_generator(self.key, self.num_items, self.num_items * 2) doc_gen["update"] = doc_generator(self.key, half_of_num_items, self.num_items) doc_gen["delete"] = doc_generator(self.key, 0, half_of_num_items) doc_gen["read"] = doc_gen["update"] for index in range(0, 4): op_type = doc_ops[index] curr_doc_gen = doc_gen[op_type] if index < 2: # Durability doc_loader for first two ops specified in doc_ops tasks.append(self.task.async_load_gen_docs( self.cluster, self.bucket, curr_doc_gen, op_type, 0, batch_size=10, process_concurrency=1, durability=self.durability_level, timeout_secs=self.sdk_timeout, sdk_client_pool=self.sdk_client_pool)) else: # Non-SyncWrites for last two ops specified in doc_ops tasks.append(self.task.async_load_gen_docs( self.cluster, self.bucket, curr_doc_gen, op_type, 0, batch_size=10, process_concurrency=1, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, sdk_client_pool=self.sdk_client_pool)) # Update num_items according to the CRUD operations self.num_items += self.num_items - half_of_num_items # Wait for all task to complete for task in tasks: # TODO: Receive failed docs and make sure only expected exceptions # are generated self.task.jython_task_manager.get_task_result(task) # Verify doc count and other stats self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items)
def test_rollback_and_persistence_race_condition(self): cluster = self.cluster gen_load = doc_generator(self.key, 0, self.num_items) for bucket in self.bucket_util.buckets: task = self.task.async_load_gen_docs( self.cluster, bucket, gen_load, "create", 0, batch_size=10, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries) self.task.jython_task_manager.get_task_result(task) # Stop persistence for server in cluster.servers[:self.nodes_init]: # Create cbepctl command object node_shell_conn = RemoteMachineShellConnection(server) cbepctl_obj = Cbepctl(node_shell_conn) for bucket in self.bucket_util.buckets: cbepctl_obj.persistence(bucket.name, "stop") # Disconnect the shell_connection node_shell_conn.disconnect() self.sleep(10, "Wait after stop_persistence") # more (non-intersecting) load gen_load = doc_generator(self.key, 0, self.num_items, doc_size=64) for bucket in self.bucket_util.buckets: task = self.task.async_load_gen_docs( self.cluster, bucket, gen_load, "create", 0, batch_size=10, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries) self.task.jython_task_manager.get_task_result(task) shell = RemoteMachineShellConnection(cluster.servers[0]) shell.kill_memcached() self.sleep(10, "Wait after kill memcached") node1_shell_conn = RemoteMachineShellConnection(cluster.servers[0]) node2_shell_conn = RemoteMachineShellConnection(cluster.servers[1]) node1_cb_stat_obj = Cbstats(node1_shell_conn) node2_cb_stat_obj = Cbstats(node2_shell_conn) node1_items = node1_cb_stat_obj.all_stats(bucket, "curr_items_tot") node2_items = node2_cb_stat_obj.all_stats(bucket, "curr_items_tot") # Disconnect the opened connections node1_shell_conn.disconnect() node2_shell_conn.disconnect() self.assertTrue(node1_items == node2_items, 'Node items not equal. Node 1:{0}, node 2:{1}' .format(node1_items, node2_items))
def test_doc_size(self): """ Insert document with empty content and max size on each available collection and validate :return: """ # Empty docs min_doc_size_gen = doc_generator("test_min_doc_size", 0, self.num_items, key_size=self.key_size, doc_size=0, mix_key_size=False, randomize_doc_size=False) # 20 MB docs max_doc_size_gen = doc_generator("test_max_doc_size", 0, self.num_items, key_size=self.key_size, doc_size=1024 * 1024 * 20, mix_key_size=False, randomize_doc_size=False) # Set to keep track of all inserted CAS values # Format know_cas[CAS] = list(vb_lists) known_cas = dict() # Client to insert docs under different collections client = SDKClient([self.cluster.master], self.bucket, compression_settings=self.sdk_compression) for doc_gen in [min_doc_size_gen, max_doc_size_gen]: while doc_gen.has_next(): key, value = doc_gen.next() for _, scope in self.bucket.scopes.items(): for _, collection in scope.collections.items(): client.select_collection(scope.name, collection.name) result = client.crud("create", key, value, self.maxttl, durability=self.durability_level, timeout=self.sdk_timeout, time_unit="seconds") if result["status"] is False: self.log_failure("Doc create failed for key '%s' " "collection::scope %s::%s - %s" % (key, scope.name, collection.name, result)) else: self.__validate_cas_for_key(key, result, known_cas) collection.num_items += 1 # Close SDK connection client.close() # Validate doc count as per bucket collections self.bucket_util.validate_docs_per_collections_all_buckets() self.validate_test_failure()
def test_dgm_to_non_dgm(self): # Prepare DGM scenario bucket = self.bucket_util.get_all_buckets()[0] num_items = self.task.load_bucket_into_dgm( self.cluster, bucket, self.key, self.num_items, self.active_resident_threshold, batch_size=10, process_concurrency=8, persist_to=self.persist_to, replicate_to=self.replicate_to) gen_create = doc_generator(self.key, num_items, num_items + self.num_items) gen_update = doc_generator(self.key, 0, self.num_items) gen_delete = doc_generator(self.key, self.num_items, num_items) # Perform continuous updates while bucket moves from DGM->non-DGM state tasks = list() tasks.append( self.task.async_load_gen_docs(self.cluster, bucket, gen_update, "update", 0, persist_to=self.persist_to, replicate_to=self.replicate_to, batch_size=10, process_concurrency=2)) tasks.append( self.task.async_load_gen_docs(self.cluster, bucket, gen_delete, "delete", 0, persist_to=self.persist_to, replicate_to=self.replicate_to, batch_size=10, process_concurrency=2)) tasks.append( self.task.async_load_gen_docs(self.cluster, bucket, gen_create, "create", 0, persist_to=self.persist_to, replicate_to=self.replicate_to, batch_size=10, process_concurrency=2)) for task in tasks: self.task.jython_task_manager.get_task_result(task)
def touch_test(self): self.log.info("Loading bucket into DGM") load_gen = doc_generator(self.key, 0, self.num_items, doc_size=self.doc_size) dgm_gen = doc_generator(self.key, self.num_items, self.num_items + 1) dgm_task = self.task.async_load_gen_docs( self.cluster, self.bucket_util.buckets[0], dgm_gen, "create", 0, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, batch_size=10, process_concurrency=4, active_resident_threshold=self.active_resident_threshold) self.task_manager.get_task_result(dgm_task) self.log.info("Touch intial self.num_items docs which are " "residing on disk due to DGM") client = SDKClient([self.cluster.master], self.bucket_util.buckets[0]) collections = BucketUtils.get_random_collections( self.bucket_util.buckets, 2, 2, 1) for self.bucket_name, scope_dict in collections.iteritems(): bucket = BucketUtils.get_bucket_obj(self.bucket_util.buckets, self.bucket_name) scope_dict = scope_dict["scopes"] for scope_name, collection_dict in scope_dict.items(): collection_dict = collection_dict["collections"] for c_name, c_data in collection_dict.items(): self.log.info("CAS test on collection %s: %s" % (scope_name, c_name)) client.select_collection(scope_name, c_name) while load_gen.has_next(): key, _ = load_gen.next() result = client.crud("touch", key, durability=self.durability_level, timeout=self.sdk_timeout) if result["status"] is not True: self.log_failure("Touch on %s failed: %s" % (key, result)) client.close() self.bucket_util._wait_for_stats_all_buckets() # Validate doc count as per bucket collections self.bucket_util.validate_docs_per_collections_all_buckets() self.validate_test_failure()
def touch_test(self): self.log.info("Loading bucket %s into %s%% DGM" % (self.bucket.name, self.active_resident_threshold)) load_gen = doc_generator(self.key, 0, self.num_items, doc_size=self.doc_size) dgm_gen = doc_generator(self.key, self.num_items, self.num_items + 1) dgm_task = self.task.async_load_gen_docs( self.cluster, self.bucket, dgm_gen, DocLoading.Bucket.DocOps.CREATE, 0, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, batch_size=10, process_concurrency=4, active_resident_threshold=self.active_resident_threshold, sdk_client_pool=self.sdk_client_pool) self.task_manager.get_task_result(dgm_task) self.log.info("Touch initial self.num_items docs which are " "residing on disk due to DGM") client = self.sdk_client_pool.get_client_for_bucket(self.bucket) collections = BucketUtils.get_random_collections([self.bucket], 2, 2, 1) for bucket_name, scope_dict in collections.iteritems(): for scope_name, collection_dict in scope_dict["scopes"].items(): for c_name, c_data in collection_dict["collections"].items(): self.log.info("CAS test on collection %s: %s" % (scope_name, c_name)) client.select_collection(scope_name, c_name) while load_gen.has_next(): key, _ = load_gen.next() result = client.crud(DocLoading.Bucket.DocOps.TOUCH, key, durability=self.durability_level, timeout=self.sdk_timeout) if result["status"] is not True: self.log_failure("Touch on %s failed: %s" % (key, result)) # change back client's scope and coll name to _default # since it was changed in the while loop to select different collection client.scope_name = CbServer.default_scope client.collection_name = CbServer.default_collection self.sdk_client_pool.release_client(client) self.validate_test_failure()
def rebalance_out_with_ops(self): gen_create = doc_generator(self.key, self.num_items, self.num_items * 2) gen_delete = doc_generator(self.key, self.num_items / 2, self.num_items) servs_out = [ self.cluster.servers[self.num_servers - i - 1] for i in range(self.nodes_out) ] tasks = list() rebalance_task = self.task.async_rebalance( self.cluster.servers[:self.nodes_init], [], servs_out) tasks_info = self.self.start_parallel_cruds(gen_create, gen_delete) self.task.jython_task_manager.get_task_result(rebalance_task) self.bucket_util.verify_doc_op_task_exceptions(tasks_info, self.cluster) self.bucket_util.log_doc_ops_task_failures(tasks_info) self.cluster.nodes_in_cluster = list( set(self.cluster.nodes_in_cluster) - set(servs_out)) for bucket in self.bucket_util.buckets: if self.doc_ops is not None: if "update" in self.doc_ops: tasks.append( self.task.async_validate_docs(self.cluster, bucket, self.gen_update, "update", 0, batch_size=10)) if "create" in self.doc_ops: tasks.append( self.task.async_validate_docs(self.cluster, bucket, gen_create, "create", 0, batch_size=10, process_concurrency=8)) if "delete" in self.doc_ops: tasks.append( self.task.async_validate_docs(self.cluster, bucket, gen_delete, "delete", 0, batch_size=10)) for task in tasks: self.task.jython_task_manager.get_task_result(task) self.bucket_util.verify_stats_all_buckets(self.num_items)
def _load_json(self, bucket, num_items, exp=0, op_type="create"): self.log.info("Creating doc_generator..") doc_create = doc_generator(self.key, 0, num_items, doc_size=self.doc_size, doc_type="json", target_vbucket=self.target_vbucket, vbuckets=self.cluster.vbuckets) self.log.info("doc_generator created") task = self.task.async_load_gen_docs( self.cluster, bucket, doc_create, op_type, exp, batch_size=10, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, compression=self.sdk_compression, sdk_client_pool=self.sdk_client_pool) self.task.jython_task_manager.get_task_result(task) self.bucket_util._wait_for_stats_all_buckets(self.cluster, self.cluster.buckets) self.bucket_util.verify_stats_all_buckets(self.cluster, self.num_items) return
def setUp(self): super(Bucket_param_test, self).setUp() self.transaction_timeout = self.input.param("transaction_timeout", 100) self.transaction_commit = self.input.param("transaction_commit", True) self.op_type = self.input.param("op_type", 'create') self.start_doc_for_insert = 0 self.key = 'test-doc'.rjust(self.key_size, '0') nodes_init = self.cluster.servers[1:self.nodes_init] \ if self.nodes_init != 1 else [] self.task.rebalance([self.cluster.master], nodes_init, []) self.cluster.nodes_in_cluster.extend( [self.cluster.master] + nodes_init) self.bucket_util.create_default_bucket( replica=self.num_replicas, compression_mode=self.compression_mode) self.bucket_util.add_rbac_user() self.src_bucket = self.bucket_util.get_all_buckets() # Reset active_resident_threshold to avoid further data load as DGM self.active_resident_threshold = 0 doc_create = doc_generator('test-d', 0, self.num_items, doc_size=self.doc_size, doc_type="json", vbuckets=self.vbuckets) for bucket in self.bucket_util.buckets: task = self.task.async_load_gen_docs( self.cluster, bucket, doc_create, "create", 0, persist_to=self.persist_to, replicate_to=self.replicate_to, batch_size=10, process_concurrency=8) self.task.jython_task_manager.get_task_result(task) # Verify initial doc load count time.sleep(20) self.log.info("==========Finished Bucket_param_test setup========")
def _dockey_data_ops(self, dockey="dockey"): target_vb = None if self.target_vbucket is not None: target_vb = [self.target_vbucket] gen_load = doc_generator(dockey, 0, self.num_items, key_size=self.key_size, doc_size=self.doc_size, doc_type=self.doc_type, vbuckets=self.cluster.vbuckets, target_vbucket=target_vb) bucket = self.bucket_util.get_all_buckets(self.cluster)[0] for op_type in ["create", "update", "delete"]: task = self.task.async_load_gen_docs( self.cluster, bucket, gen_load, op_type, 0, batch_size=20, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, sdk_client_pool=self.sdk_client_pool) self.task.jython_task_manager.get_task_result(task) if op_type == "delete": self.num_items = 0 self._persist_and_verify()
def setUp(self): super(OpsChangeCasTests, self).setUp() self.key = "test_cas" self.expire_time = self.input.param("expire_time", 35) self.item_flag = self.input.param("item_flag", 0) self.load_gen = doc_generator(self.key, 0, self.num_items, doc_size=self.doc_size) self.node_data = dict() for node in self.cluster_util.get_kv_nodes(): shell = RemoteMachineShellConnection(node) cb_stat = Cbstats(shell) self.node_data[node.ip] = dict() self.node_data[node.ip]["shell"] = shell self.node_data[node.ip]["cb_stat"] = Cbstats(shell) self.node_data[node.ip]["active"] = cb_stat.vbucket_list( self.bucket, "active") self.node_data[node.ip]["replica"] = cb_stat.vbucket_list( self.bucket, "replica") if self.sdk_client_pool: self.client = self.sdk_client_pool.get_client_for_bucket( self.bucket) else: self.client = SDKClient([self.cluster.master], self.bucket)
def setUp(self): super(CollectionsRebalance, self).setUp() self.bucket_util._expiry_pager() self.load_gen = doc_generator(self.key, 0, self.num_items) self.bucket = self.bucket_util.buckets[0] self.rest = RestConnection(self.cluster.master) self.data_load_spec = self.input.param("data_load_spec", "volume_test_load") self.data_load_stage = self.input.param("data_load_stage", "before") self.data_load_type = self.input.param("data_load_type", "async") self.nodes_swap = self.input.param("nodes_swap", 1) self.nodes_failover = self.input.param("nodes_failover", 1) self.failover_ops = [ "graceful_failover_rebalance_out", "hard_failover_rebalance_out", "graceful_failover_recovery", "hard_failover_recovery" ] self.step_count = self.input.param("step_count", -1) self.replicas_for_failover = self.input.param("replicas_for_failover", 3) self.recovery_type = self.input.param("recovery_type", "full") self.compaction = self.input.param("compaction", False) self.warmup = self.input.param("warmup", False) self.update_replica = self.input.param( "update_replica", False) # for replica + rebalance tests self.num_replicas = self.input.param( "num_replicas", 1) # for replica + rebalance tests if (self.compaction): self.compaction_tasks = list()
def setUp(self): super(CollectionsRebalance, self).setUp() self.bucket_util._expiry_pager() self.load_gen = doc_generator(self.key, 0, self.num_items) self.bucket = self.bucket_util.buckets[0] self.rest = RestConnection(self.cluster.master) self.data_load_spec = self.input.param("data_load_spec", "volume_test_load") self.data_load_stage = self.input.param("data_load_stage", "before") self.data_load_type = self.input.param("data_load_type", "async") self.nodes_swap = self.input.param("nodes_swap", 1) self.nodes_failover = self.input.param("nodes_failover", 1) self.failover_ops = [ "graceful_failover_rebalance_out", "hard_failover_rebalance_out", "graceful_failover_recovery", "hard_failover_recovery" ] self.step_count = self.input.param("step_count", -1) self.recovery_type = self.input.param("recovery_type", "full") self.compaction = self.input.param("compaction", False) if self.compaction: self.disable_auto_compaction() self.warmup = self.input.param("warmup", False) self.update_replica = self.input.param( "update_replica", False) # for replica + rebalance tests self.updated_num_replicas = self.input.param( "updated_num_replicas", 1) # for replica + rebalance tests, forced hard failover self.forced_hard_failover = self.input.param( "forced_hard_failover", False) # for forced hard failover tests self.change_ram_quota_cluster = self.input.param( "change_ram_quota_cluster", False) # To change during rebalance self.skip_validations = self.input.param("skip_validations", True) if self.compaction: self.compaction_tasks = list() self.dgm_test = self.input.param("dgm_test", False)
def setUp(self): super(Bucket_DGM_Tests, self).setUp() self.key = 'test_docs'.rjust(self.key_size, '0') nodes_init = self.cluster.servers[1:self.nodes_init] \ if self.nodes_init != 1 else [] self.task.rebalance([self.cluster.master], nodes_init, []) self.cluster.nodes_in_cluster.extend( [self.cluster.master] + nodes_init) self.bucket_util.create_default_bucket( ram_quota=self.bucket_size, replica=self.num_replicas, maxTTL=self.maxttl, compression_mode=self.compression_mode) self.bucket_util.add_rbac_user() self.cluster_util.print_cluster_stats() doc_create = doc_generator( self.key, 0, self.num_items, doc_size=self.doc_size, doc_type=self.doc_type, vbuckets=self.vbuckets) for bucket in self.bucket_util.buckets: task = self.task.async_load_gen_docs( self.cluster, bucket, doc_create, "create", 0, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, batch_size=10, process_concurrency=8) self.task.jython_task_manager.get_task_result(task) # Verify initial doc load count self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) self.log.info("========= Finished Bucket_DGM_Tests setup =======")
def setUp(self): super(RebalanceStartStopTests, self).setUp() extra_nodes_in = self.input.param("extra_nodes_in", 0) extra_nodes_out = self.input.param("extra_nodes_out", 0) self.servs_init = self.servers[:self.nodes_init] self.servs_in = [ self.servers[i + self.nodes_init] for i in range(self.nodes_in) ] self.servs_out = [ self.servers[self.nodes_init - i - 1] for i in range(self.nodes_out) ] self.extra_servs_in = [ self.servers[i + self.nodes_init + self.nodes_in] for i in range(extra_nodes_in) ] self.extra_servs_out = [ self.servers[self.nodes_init - i - 1 - self.nodes_out] for i in range(extra_nodes_out) ] self.withMutationOps = self.input.param("withMutationOps", True) self.sleep_before_rebalance = self.input.param( "sleep_before_rebalance", 0) if self.spec_name is not None: self.num_items = 20000 self.items = 20000 # We need to use "test_collections" key for update, # since doc_loading was done from spec self.gen_update = doc_generator("test_collections", 0, (self.items / 2), mutation_type="SET")
def test_MB_34947(self): # Update already Created docs with async_writes load_gen = doc_generator(self.key, 0, self.num_items, key_size=self.key_size, doc_size=self.doc_size, doc_type=self.doc_type, vbuckets=self.cluster.vbuckets) task = self.task.async_load_gen_docs( self.cluster, self.def_bucket, load_gen, "update", 0, persist_to=self.persist_to, replicate_to=self.replicate_to, timeout_secs=self.sdk_timeout, batch_size=10, process_concurrency=8) self.task.jython_task_manager.get_task_result(task) # Update bucket replica to new value bucket_helper = BucketHelper(self.cluster.master) bucket_helper.change_bucket_props( self.def_bucket, replicaNumber=self.new_replica) self.bucket_util.print_bucket_stats(self.cluster) # Start rebalance task rebalance = self.task.async_rebalance(self.cluster.servers, [], []) self.sleep(10, "Wait for rebalance to start") # Wait for rebalance task to complete self.task.jython_task_manager.get_task_result(rebalance) # Assert if rebalance failed self.assertTrue(rebalance.result, "Rebalance failed after replica update")
def test_max_key_size(self): if self.use_default_collection: self.key_size = 251 self.collection_name = CbServer.default_collection else: self.key_size = 247 self.collection_name = "collection-1" BucketUtils.create_collection( self.cluster.master, self.bucket, scope_name=CbServer.default_scope, collection_spec={"name": self.collection_name}) gen_load = doc_generator("test-max-key-size", 0, 1, key_size=self.key_size, vbuckets=self.cluster.vbuckets) task = self.task.async_load_gen_docs(self.cluster, self.bucket, gen_load, "create", self.maxttl, batch_size=20, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, collection=self.collection_name) self.task.jython_task_manager.get_task_result(task) if task.fail: self.log.info("Inserting doc key > max size failed as expected") else: self.fail("Inserting doc key greater than max key size " "succeeded when it should have failed")
def load_docs_in_cb_bucket_before_and_after_cbas_connect(self): self.setup_for_test() # Load more docs in Couchbase bucket. self.perform_doc_ops_in_all_cb_buckets("create", self.num_items, self.num_items * 2) self.bucket_util.verify_stats_all_buckets(self.num_items * 2) if self.test_abort_snapshot: self.log.info("Creating sync_write aborts after dataset connect") for server in self.cluster_util.get_kv_nodes(): ssh_shell = RemoteMachineShellConnection(server) cbstats = Cbstats(ssh_shell) replica_vbs = cbstats.vbucket_list( self.bucket_util.buckets[0].name, "replica") load_gen = doc_generator("test_abort_key", self.num_items, self.num_items, target_vbucket=replica_vbs) success = self.bucket_util.load_durable_aborts( ssh_shell, [load_gen], self.bucket_util.buckets[0], self.durability_level, "update", "all_aborts") if not success: self.log_failure("Simulating aborts failed") ssh_shell.disconnect() self.validate_test_failure() # Validate no. of items in CBAS dataset if not self.cbas_util.validate_cbas_dataset_items_count( self.cbas_dataset_name, self.num_items * 2): self.fail("No. of items in CBAS dataset do not match " "that in the CB bucket")
def setUp(self): super(RebalanceStartStopTests, self).setUp() extra_nodes_in = self.input.param("extra_nodes_in", 0) extra_nodes_out = self.input.param("extra_nodes_out", 0) self.servs_init = self.servers[:self.nodes_init] self.servs_in = [self.servers[i + self.nodes_init] for i in range(self.nodes_in)] self.servs_out = [self.servers[self.nodes_init - i - 1] for i in range(self.nodes_out)] self.extra_servs_in = [self.servers[i + self.nodes_init + self.nodes_in] for i in range(extra_nodes_in)] self.extra_servs_out = [self.servers[self.nodes_init - i - 1 - self.nodes_out] for i in range(extra_nodes_out)] self.withMutationOps = self.input.param("withMutationOps", True) self.sleep_before_rebalance = self.input.param("sleep_before_rebalance", 0) if self.spec_name is not None: self.num_items = 20000 self.items = 20000 init_doc_load_spec = \ self.bucket_util.get_crud_template_from_package("initial_load") # Using the same key as defined in the loading spec self.gen_update = doc_generator( init_doc_load_spec["doc_crud"][ MetaCrudParams.DocCrud.COMMON_DOC_KEY], 0, (self.items / 2), mutation_type="SET") shell = RemoteMachineShellConnection(self.cluster.master) shell.enable_diag_eval_on_non_local_hosts() shell.disconnect()
def key_not_exists_test(self): client = SDKClient([self.cluster.master], self.bucket) collections = BucketUtils.get_random_collections([self.bucket], 1, 1, 1) scope_dict = collections[self.bucket.name]["scopes"] scope_name = scope_dict.keys()[0] collection_name = scope_dict[scope_name]["collections"].keys()[0] client.select_collection(scope_name, collection_name) self.log.info("CAS test on collection %s: %s" % (scope_name, collection_name)) load_gen = doc_generator(self.key, 0, self.num_items, doc_size=256) key, val = load_gen.next() for _ in range(1500): result = client.crud("create", key, val, durability=self.durability_level, timeout=self.sdk_timeout) if result["status"] is False: self.log_failure("Create failed: %s" % result) create_cas = result["cas"] # Delete and verify get fails result = client.crud("delete", key, durability=self.durability_level, timeout=self.sdk_timeout) if result["status"] is False: self.log_failure("Delete failed: %s" % result) elif result["cas"] <= create_cas: self.log_failure("Delete returned invalid cas: %s" % result) result = client.crud("read", key, timeout=self.sdk_timeout) if result["status"] is True: self.log_failure("Read succeeded after delete: %s" % result) elif SDKException.DocumentNotFoundException \ not in str(result["error"]): self.log_failure("Invalid exception during read " "for non-exists key: %s" % result) # cas errors do not sleep the test for 10 seconds, # plus we need to check that the correct error is being thrown result = client.crud("replace", key, val, exp=60, timeout=self.sdk_timeout, cas=create_cas) if result["status"] is True: self.log_failure("Replace succeeded after delete: %s" % result) if SDKException.DocumentNotFoundException \ not in str(result["error"]): self.log_failure("Invalid exception during read " "for non-exists key: %s" % result) # Validate doc count as per bucket collections self.bucket_util.validate_docs_per_collections_all_buckets() self.validate_test_failure()
def _dockey_tap(self, dockey="dockey"): gen_load = doc_generator(dockey, 0, self.num_items, doc_type="json") bucket = self.bucket_util.get_all_buckets()[0] task = self.task.async_load_gen_docs(self.cluster, bucket, gen_load, "create", 0, batch_size=20, persist_to=self.persist_to, replicate_to=self.replicate_to, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries) self.task.jython_task_manager.get_task_result(task) self._persist_and_verify() # assert if there are not enough nodes to failover rest = RestConnection(self.cluster.master) num_nodes = len(rest.node_statuses()) self.assertTrue(num_nodes > 1, "ERROR: Not enough nodes to do failover") # failover 1 node(we have 1 replica) and verify the keys self.cluster.failover(self.servers[:num_nodes], self.servers[(num_nodes - 1):num_nodes]) self.nodes_init -= 1 self._persist_and_verify()
def custom_load(self, maxttl=0): self.key = "test_collections" start = self.bucket.scopes[CbServer.default_scope] \ .collections[CbServer.default_collection] \ .num_items load_gen = doc_generator(self.key, start, start + 1) tasks = [] tasks.append( self.task.async_load_gen_docs( self.cluster, self.bucket, load_gen, "create", maxttl, batch_size=1000, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, active_resident_threshold=self.dgm, compression=self.sdk_compression, timeout_secs=self.sdk_timeout, scope=CbServer.default_scope, collection=CbServer.default_collection)) for task in tasks: self.task.jython_task_manager.get_task_result(task) if task.fail: self.fail("preload dgm failed") self.bucket_util.print_bucket_stats()
def load_bucket_into_dgm(self, cluster, bucket, key, num_items, active_resident_threshold, load_batch_size=20000, batch_size=10, process_concurrency=4, persist_to=None, replicate_to=None): rest = BucketHelper(cluster.master) bucket_stat = rest.get_bucket_stats_for_node(bucket.name, cluster.master) while bucket_stat["vb_active_resident_items_ratio"] > \ active_resident_threshold: gen_load = doc_generator(key, num_items, num_items + load_batch_size, doc_type="binary") num_items += load_batch_size task = self.async_load_gen_docs( cluster, bucket, gen_load, "create", 0, batch_size=batch_size, process_concurrency=process_concurrency, persist_to=persist_to, replicate_to=replicate_to) self.jython_task_manager.get_task_result(task) bucket_stat = rest.get_bucket_stats_for_node( bucket.name, cluster.master) return num_items
def rebalance_out_with_warming_up(self): master_restart = self.input.param("master_restart", False) if master_restart: warmup_node = self.cluster.master else: warmup_node = self.cluster.servers[len(self.cluster.servers) - self.nodes_out - 1] servs_out = self.cluster.servers[len(self.cluster.servers) - self.nodes_out:] if self.test_abort_snapshot: self.log.info("Creating sync_write abort scenario for replica vbs") for server in self.cluster_util.get_kv_nodes(self.cluster): ssh_shell = RemoteMachineShellConnection(server) cbstats = Cbstats(ssh_shell) replica_vbs = cbstats.vbucket_list( self.cluster.buckets[0].name, "replica") load_gen = doc_generator(self.key, 0, 5000, target_vbucket=replica_vbs) success = self.bucket_util.load_durable_aborts( ssh_shell, [load_gen], self.cluster.buckets[0], self.durability_level, "update", "all_aborts") if not success: self.log_failure("Simulating aborts failed") ssh_shell.disconnect() self.validate_test_failure() shell = RemoteMachineShellConnection(warmup_node) shell.stop_couchbase() self.sleep(20) shell.start_couchbase() shell.disconnect() # Workaround for Eph case (MB-44682 - Not a bug) if self.bucket_type == Bucket.Type.EPHEMERAL: self.sleep(15, "Wait for couchbase server to start") rebalance = self.task.async_rebalance( self.cluster.servers, [], servs_out) self.task.jython_task_manager.get_task_result(rebalance) self.assertTrue(rebalance.result, "Rebalance Failed") self.cluster.nodes_in_cluster = list(set(self.cluster.nodes_in_cluster) - set(servs_out)) if rebalance.result is False: self.log.info("Rebalance was failed as expected") self.assertTrue(self.bucket_util._wait_warmup_completed( self.cluster_util.get_kv_nodes(self.cluster), self.cluster.buckets[0], wait_time=self.wait_timeout * 10)) self.log.info("Second attempt to rebalance") rebalance = self.task.async_rebalance( self.cluster.servers, [], servs_out) self.task.jython_task_manager.get_task_result(rebalance) self.assertTrue(rebalance.result, "Rebalance attempt failed again") self.cluster.nodes_in_cluster = list(set(self.cluster.nodes_in_cluster) - set(servs_out)) if not self.atomicity: self.bucket_util.verify_cluster_stats(self.cluster, self.num_items, timeout=self.wait_timeout) self.bucket_util.verify_unacked_bytes_all_buckets(self.cluster)
def setUp(self): super(MagmaFailures, self).setUp() self.gen_create = doc_generator( self.key, 0, self.num_items, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) self.result_task = self._load_all_buckets(self.cluster, self.gen_create, "create", 0, batch_size=self.batch_size, dgm_batch=self.dgm_batch) if self.active_resident_threshold != 100: for task in self.result_task.keys(): self.num_items = task.doc_index self.log.info("Verifying num_items counts after doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats()