def test_replica_update(self): if self.atomicity: replica_count = 3 else: replica_count = 4 if self.nodes_init < 2: self.log.error("Test not supported for < 2 node cluster") return doc_ops = self.input.param("doc_ops", "") bucket_helper = BucketHelper(self.cluster.master) doc_count = self.num_items start_doc_for_insert = self.num_items self.is_sync_write_enabled = DurabilityHelper.is_sync_write_enabled( self.bucket_durability_level, self.durability_level) # Replica increment tests doc_count, start_doc_for_insert = self.generic_replica_update( doc_count, doc_ops, bucket_helper, range(1, min(replica_count, self.nodes_init)), start_doc_for_insert) # Replica decrement tests _, _ = self.generic_replica_update( doc_count, doc_ops, bucket_helper, range(min(replica_count, self.nodes_init)-2, -1, -1), start_doc_for_insert)
def test_rebalance_out_durabilitybreaks_rebalance_in(self): self.assertTrue(self.num_replicas >= 1, "Need at-least one replica to run this test") durability_will_fail = False def_bucket = self.cluster.buckets[0] nodes_in_cluster = self.nodes_init nodes_required_for_durability = int( floor((self.num_replicas + 1) / 2) + 1) gen_update = doc_generator(self.key, 0, self.num_items) num_of_docs_to_insert = 1000 # Rebalance all nodes expect master one-by-one for _ in range(self.nodes_init - 1): rebalance_result = self.task.rebalance( self.cluster.servers[:nodes_in_cluster], [], [self.cluster.servers[nodes_in_cluster - 1]]) self.assertTrue(rebalance_result) nodes_in_cluster -= 1 if nodes_in_cluster < nodes_required_for_durability: if DurabilityHelper.is_sync_write_enabled( self.bucket_durability_level, self.durability_level): durability_will_fail = True tasks = list() # Perform CRUD operations after rebalance_out to verify the # durability outcome gen_create = doc_generator(self.key, self.num_items, self.num_items + num_of_docs_to_insert) if self.atomicity: tasks.append( self.task.async_load_gen_docs_atomicity( self.cluster, self.cluster.buckets, gen_create, "rebalance_update", exp=0, batch_size=10, process_concurrency=self.process_concurrency, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, update_count=self.update_count, transaction_timeout=self.transaction_timeout, commit=True, durability=self.durability_level, sync=True, num_threads=1, record_fail=True, defer=self.defer)) self.num_items += num_of_docs_to_insert def_bucket.scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items += num_of_docs_to_insert else: tasks.append( self.task.async_load_gen_docs( self.cluster, def_bucket, gen_create, "create", exp=0, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, batch_size=20, process_concurrency=4, sdk_client_pool=self.sdk_client_pool)) tasks.append( self.task.async_load_gen_docs( self.cluster, def_bucket, gen_update, "update", 0, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, batch_size=20, process_concurrency=4, sdk_client_pool=self.sdk_client_pool)) # Wait for all CRUD tasks to complete for task in tasks: self.task.jython_task_manager.get_task_result(task) if durability_will_fail: self.log.info( "Durability broken with cluster {0}, replica {1}".format( self.cluster.servers[:nodes_in_cluster], self.num_replicas)) assert_msg = \ "Items crud succeeded although majority condition failed." if not self.atomicity: for task in tasks: if task.op_type == "create": self.assertTrue( len(task.fail) == num_of_docs_to_insert, assert_msg) elif task.op_type == "update": self.assertTrue( len(task.fail) == (gen_update.end - gen_update.start), assert_msg) break elif not self.atomicity: # If durability works fine, re-calculate the self.num_items self.num_items += num_of_docs_to_insert def_bucket.scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items += num_of_docs_to_insert # Reset the tasks list to reuse tasks = list() # Create tasks for doc verification tasks.append( self.task.async_validate_docs( self.cluster, def_bucket, gen_update, "update", 0, batch_size=10, process_concurrency=4, sdk_client_pool=self.sdk_client_pool)) tasks.append( self.task.async_validate_docs( self.cluster, def_bucket, gen_create, "create", 0, batch_size=10, process_concurrency=4, sdk_client_pool=self.sdk_client_pool)) # Wait for all verification tasks to complete for task in tasks: self.task.jython_task_manager.get_task_result(task) # Rebalance-in single node back into the cluster rebalance_result = self.task.rebalance( self.cluster.servers[:nodes_in_cluster], self.cluster.servers[nodes_in_cluster:nodes_in_cluster + 1], []) # Wait for rebalance-in task to complete self.assertTrue(rebalance_result) # Reset the tasks list tasks = list() # Perform CRUD operations after rebalance_in to verify the # durability is working as expected again gen_create = doc_generator(self.key, self.num_items, self.num_items + num_of_docs_to_insert) if self.atomicity: tasks.append( self.task.async_load_gen_docs_atomicity( self.cluster, self.cluster.buckets, gen_create, "rebalance_update", exp=0, batch_size=10, process_concurrency=self.process_concurrency, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, update_count=self.update_count, transaction_timeout=self.transaction_timeout, commit=True, durability=self.durability_level, sync=True, num_threads=1, record_fail=True, defer=self.defer)) else: tasks.append( self.task.async_load_gen_docs( self.cluster, def_bucket, gen_create, "create", exp=0, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, batch_size=20, sdk_client_pool=self.sdk_client_pool)) tasks.append( self.task.async_load_gen_docs( self.cluster, def_bucket, gen_update, "update", 0, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, batch_size=20, sdk_client_pool=self.sdk_client_pool)) for task in tasks: self.task.jython_task_manager.get_task_result(task) # Wait for all CRUD tasks to complete if not self.atomicity: assert_msg = \ "Items crud succeeded although majority condition failed." for task in tasks: if task.op_type == "create": self.assertTrue(len(task.fail) == 0, assert_msg) elif task.op_type == "update": self.assertTrue(len(task.fail) == 0, assert_msg) # After rebalance-in durability should work fine # So recalculating the self.num_items to match gen_create loader self.num_items += num_of_docs_to_insert def_bucket.scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items += num_of_docs_to_insert # Reset the tasks list tasks = list() # Create tasks for doc verification tasks.append( self.task.async_validate_docs( self.cluster, def_bucket, gen_update, "update", 0, batch_size=10, process_concurrency=4, sdk_client_pool=self.sdk_client_pool)) tasks.append( self.task.async_validate_docs( self.cluster, def_bucket, gen_create, "create", 0, batch_size=10, process_concurrency=4, sdk_client_pool=self.sdk_client_pool)) # Wait for all doc verification tasks to complete for task in tasks: self.task.jython_task_manager.get_task_result(task) # Verify doc load count to match the overall CRUDs self.bucket_util._wait_for_stats_all_buckets( self.cluster, self.cluster.buckets) self.bucket_util.validate_docs_per_collections_all_buckets( self.cluster)
def test_collection_not_exists(self): """ 1. Load docs into required collection 2. Validate docs based on the targeted collection 3. Create non-default scope/collection for CRUDs to happen 4. Perform doc_ops again and perform CRUDs 5. Drop the target collection and validate the CollectionNotExists exception from client side 6. Recreate non-default collection and re-create the docs and validate """ def validate_vb_detail_stats(): failed = durability_helper.verify_vbucket_details_stats( self.bucket, self.cluster_util.get_kv_nodes(self.cluster), vbuckets=self.cluster.vbuckets, expected_val=verification_dict) if failed: self.log_failure("vBucket_details validation failed") self.bucket_util.validate_docs_per_collections_all_buckets( self.cluster) sync_write_enabled = DurabilityHelper.is_sync_write_enabled( self.bucket_durability_level, self.durability_level) num_cols_in_bucket = 0 for _, scope in self.bucket.scopes.items(): for _, _ in scope.collections.items(): num_cols_in_bucket += 1 verification_dict = dict() verification_dict["ops_create"] = num_cols_in_bucket * self.num_items verification_dict["ops_update"] = 0 verification_dict["ops_delete"] = 0 verification_dict["rollback_item_count"] = 0 verification_dict["sync_write_aborted_count"] = 0 verification_dict["sync_write_committed_count"] = 0 durability_helper = DurabilityHelper(self.log, len(self.cluster.kv_nodes), durability=self.durability_level) drop_scope = self.input.param("drop_scope", False) if self.scope_name != CbServer.default_scope: self.scope_name = self.bucket_util.get_random_name() if self.collection_name != CbServer.default_collection: self.collection_name = self.bucket_util.get_random_name() # Doc generator used for mutations doc_gen = doc_generator("test_col_not_exists", 0, 10) # Acquire SDK client for mutations client = self.sdk_client_pool.get_client_for_bucket( self.bucket, self.scope_name, self.collection_name) doc_ttl, _ = \ SDKExceptionTests.__get_random_doc_ttl_and_durability_level() self.log.info( "Creating docs with doc_ttl %s into %s:%s:%s" % (doc_ttl, self.bucket.name, self.scope_name, self.collection_name)) retry_reason = SDKException.RetryReason while doc_gen.has_next(): key, value = doc_gen.next() result = client.crud("create", key, value, exp=doc_ttl, durability=self.durability_level, timeout=30) if self.collection_name == CbServer.default_collection: if result["status"] is False: self.log_failure("Create doc failed for key: %s" % key) else: verification_dict["ops_create"] += 1 if sync_write_enabled: verification_dict["sync_write_committed_count"] += 1 self.bucket.scopes[self.scope_name].collections[ self.collection_name].num_items += 1 elif result["status"] is True: self.log_failure("Create didn't fail as expected for key: %s" % key) elif (SDKException.AmbiguousTimeoutException not in str(result["error"]) or retry_reason.COLLECTION_NOT_FOUND not in str(result["error"])) \ and ( SDKException.RequestCanceledException not in str(result["error"]) or retry_reason.COLLECTION_MAP_REFRESH_IN_PROGRESS not in str(result["error"])): self.log_failure("Invalid exception for key %s: %s" % (key, result["error"])) validate_vb_detail_stats() # Create required scope/collection for successful CRUD operation self.create_scope_collection() # Reset doc_gen itr value for retry purpose doc_gen.reset() doc_ttl, _ = \ SDKExceptionTests.__get_random_doc_ttl_and_durability_level() self.log.info( "Creating docs with doc_ttl %s into %s:%s:%s" % (doc_ttl, self.bucket.name, self.scope_name, self.collection_name)) op_type = "create" if self.collection_name == CbServer.default_collection: op_type = "update" while doc_gen.has_next(): key, value = doc_gen.next() result = client.crud(op_type, key, value, exp=doc_ttl, durability=self.durability_level) if result["status"] is False: self.log_failure("Create fail for key %s: %s" % (key, result)) else: if op_type == "create": verification_dict["ops_create"] += 1 self.bucket.scopes[self.scope_name].collections[ self.collection_name].num_items += 1 else: verification_dict["ops_update"] += 1 if sync_write_enabled: verification_dict["sync_write_committed_count"] += 1 validate_vb_detail_stats() self.validate_test_failure() if drop_scope: self.log.info("Dropping scope %s" % self.scope_name) self.bucket_util.drop_scope(self.cluster.master, self.bucket, self.scope_name) else: self.log.info("Dropping collection %s:%s" % (self.scope_name, self.collection_name)) self.bucket_util.drop_collection(self.cluster.master, self.bucket, self.scope_name, self.collection_name) validate_vb_detail_stats() self.validate_test_failure() # Reset doc_gen itr value for retry purpose doc_gen.reset() while doc_gen.has_next(): key, value = doc_gen.next() result = client.crud("create", key, value, exp=doc_ttl, durability=self.durability_level) if result["status"] is True: self.log_failure("Create doc succeeded for dropped collection") validate_vb_detail_stats() self.validate_test_failure() # Re-create the dropped collection self.create_scope_collection(create_scope=drop_scope) if self.collection_name != CbServer.default_collection: doc_gen.reset() while doc_gen.has_next(): key, value = doc_gen.next() result = client.crud("create", key, value, exp=doc_ttl, durability=self.durability_level) if result["status"] is False: self.log_failure("Create failed after collection recreate " "for key %s: %s" % (key, result["error"])) else: verification_dict["ops_create"] += 1 if sync_write_enabled: verification_dict["sync_write_committed_count"] += 1 self.bucket.scopes[self.scope_name].collections[ self.collection_name].num_items += 1 validate_vb_detail_stats() # Release the acquired client self.sdk_client_pool.release_client(client) self.validate_test_failure()
def setUp(self): super(CrashTest, self).setUp() self.doc_ops = self.input.param("doc_ops", None) self.process_name = self.input.param("process", None) self.service_name = self.input.param("service", "data") self.sig_type = self.input.param("sig_type", "SIGKILL").upper() self.target_node = self.input.param("target_node", "active") self.pre_warmup_stats = {} self.timeout = 120 self.new_docs_to_add = 10000 if self.doc_ops is not None: self.doc_ops = self.doc_ops.split(";") if not self.atomicity: self.durability_helper = DurabilityHelper( self.log, self.nodes_init, durability=self.durability_level, replicate_to=self.replicate_to, persist_to=self.persist_to) self.bucket_util.create_default_bucket( cluster=self.cluster, bucket_type=self.bucket_type, ram_quota=self.bucket_size, replica=self.num_replicas, compression_mode="off", storage=self.bucket_storage, eviction_policy=self.bucket_eviction_policy) self.bucket_util.add_rbac_user(self.cluster.master) if self.sdk_client_pool: self.log.info("Creating SDK clients for client_pool") for bucket in self.cluster.buckets: self.sdk_client_pool.create_clients( bucket, [self.cluster.master], self.sdk_pool_capacity, compression_settings=self.sdk_compression) self.__is_sync_write_enabled = DurabilityHelper.is_sync_write_enabled( self.bucket_durability_level, self.durability_level) verification_dict = dict() verification_dict["ops_create"] = self.num_items verification_dict["sync_write_aborted_count"] = 0 verification_dict["rollback_item_count"] = 0 verification_dict["pending_writes"] = 0 if self.__is_sync_write_enabled: verification_dict["sync_write_committed_count"] = self.num_items # Load initial documents into the buckets self.log.info("Loading initial documents") gen_create = doc_generator(self.key, 0, self.num_items, key_size=self.key_size, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster.vbuckets) if self.atomicity: task = self.task.async_load_gen_docs_atomicity( self.cluster, self.cluster.buckets, gen_create, "create", exp=0, batch_size=10, process_concurrency=self.process_concurrency, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, update_count=self.update_count, transaction_timeout=self.transaction_timeout, commit=True, sync=self.sync) self.task.jython_task_manager.get_task_result(task) else: for bucket in self.cluster.buckets: task = self.task.async_load_gen_docs( self.cluster, bucket, gen_create, DocLoading.Bucket.DocOps.CREATE, self.maxttl, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, batch_size=10, process_concurrency=8, sdk_client_pool=self.sdk_client_pool) self.task.jython_task_manager.get_task_result(task) self.bucket_util._wait_for_stats_all_buckets( self.cluster, self.cluster.buckets) # Verify cbstats vbucket-details stats_failed = \ self.durability_helper.verify_vbucket_details_stats( bucket, self.cluster_util.get_kv_nodes(self.cluster), vbuckets=self.cluster.vbuckets, expected_val=verification_dict) if stats_failed: self.fail("Cbstats verification failed") self.bucket_util.verify_stats_all_buckets(self.cluster, self.num_items) self.cluster_util.print_cluster_stats(self.cluster) self.bucket_util.print_bucket_stats(self.cluster) self.log.info("==========Finished CrashTest setup========")
def setUp(self): super(BasicOps, self).setUp() self.is_sync_write_enabled = DurabilityHelper.is_sync_write_enabled( self.bucket_durability_level, self.durability_level) self.log.info("==========Finished BasisOps base setup========")
def setUp(self): super(CrashTest, self).setUp() self.doc_ops = self.input.param("doc_ops", None) self.process_name = self.input.param("process", None) self.service_name = self.input.param("service", "data") self.sig_type = self.input.param("sig_type", "SIGKILL").upper() self.target_node = self.input.param("target_node", "active") self.client_type = self.input.param("client_type", "sdk").lower() self.N1qltxn = self.input.param("N1qltxn", False) self.allowed_hosts = self.input.param("allowed_hosts", False) self.pre_warmup_stats = dict() self.timeout = 120 self.new_docs_to_add = 10000 if self.doc_ops is not None: self.doc_ops = self.doc_ops.split(";") if not self.atomicity: self.durability_helper = DurabilityHelper( self.log, self.nodes_init, durability=self.durability_level, replicate_to=self.replicate_to, persist_to=self.persist_to) self.__is_sync_write_enabled = DurabilityHelper.is_sync_write_enabled( self.bucket_durability_level, self.durability_level) verification_dict = dict() verification_dict["ops_create"] = \ self.cluster.buckets[0].scopes[ CbServer.default_scope].collections[ CbServer.default_collection].num_items verification_dict["sync_write_aborted_count"] = 0 verification_dict["rollback_item_count"] = 0 verification_dict["pending_writes"] = 0 if self.__is_sync_write_enabled: verification_dict["sync_write_committed_count"] = \ verification_dict["ops_create"] # Load initial documents into the buckets transaction_gen_create = doc_generator( "transaction_key", 0, self.num_items, key_size=self.key_size, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster.vbuckets) gen_create = doc_generator(self.key, 0, self.num_items, key_size=self.key_size, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster.vbuckets) if self.atomicity: transaction_task = self.task.async_load_gen_docs_atomicity( self.cluster, self.cluster.buckets, transaction_gen_create, DocLoading.Bucket.DocOps.CREATE, exp=0, batch_size=10, process_concurrency=self.process_concurrency, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, update_count=self.update_count, transaction_timeout=self.transaction_timeout, commit=True, sync=self.sync) self.task.jython_task_manager.get_task_result(transaction_task) for bucket in self.cluster.buckets: task = self.task.async_load_gen_docs( self.cluster, bucket, gen_create, DocLoading.Bucket.DocOps.CREATE, self.maxttl, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, batch_size=10, process_concurrency=8) self.task.jython_task_manager.get_task_result(task) self.bucket_util._wait_for_stats_all_buckets( self.cluster, self.cluster.buckets) self.cluster.buckets[0].scopes[CbServer.default_scope].collections[ CbServer.default_collection].num_items += self.num_items verification_dict["ops_create"] += self.num_items if self.__is_sync_write_enabled \ and self.durability_level != Bucket.DurabilityLevel.NONE: verification_dict["sync_write_committed_count"] += \ self.num_items # Verify cbstats vbucket-details stats_failed = self.durability_helper.verify_vbucket_details_stats( bucket, self.cluster_util.get_kv_nodes(self.cluster), vbuckets=self.cluster.vbuckets, expected_val=verification_dict) if self.atomicity is False: if stats_failed: self.fail("Cbstats verification failed") self.bucket_util.verify_stats_all_buckets( self.cluster, self.cluster.buckets[0].scopes[CbServer.default_scope]. collections[CbServer.default_collection].num_items) self.bucket = self.cluster.buckets[0] if self.N1qltxn: self.n1ql_server = self.cluster_util.get_nodes_from_services_map( cluster=self.cluster, service_type=CbServer.Services.N1QL, get_all_nodes=True) self.n1ql_helper = N1QLHelper(server=self.n1ql_server, use_rest=True, buckets=self.cluster.buckets, log=self.log, scan_consistency='REQUEST_PLUS', num_collection=3, num_buckets=1, num_savepoints=1, override_savepoint=False, num_stmt=10, load_spec=self.data_spec_name) self.bucket_col = self.n1ql_helper.get_collections() self.stmts = self.n1ql_helper.get_stmt(self.bucket_col) self.stmts = self.n1ql_helper.create_full_stmts(self.stmts) self.log.info("==========Finished CrashTest setup========")