def test_crash_process(self): """ 1. Starting loading docs into the default bucket 2. Crash the requested process, which will not impact the memcached operations 3. Wait for load bucket task to complete 4. Validate the docs for durability """ def_bucket = self.cluster.buckets[0] target_node = self.getTargetNode() remote = RemoteMachineShellConnection(target_node) target_vbuckets = range(0, self.cluster_util.vbuckets) retry_exceptions = list() self.transaction_load_task = None self.doc_loading_task = None self.N1ql_load_task = None # If Memcached is killed, we should not perform KV ops on # particular node. If not we can target all nodes for KV operation. if self.process_name == "memcached": target_vbuckets = CrashTest.getVbucketNumbers( remote, def_bucket.name, self.target_node) if self.target_node == "active": retry_exceptions = [SDKException.TimeoutException] if len(target_vbuckets) == 0: self.log.error("No target vbucket list generated to load data") remote.disconnect() return bucket_dict = BucketUtils.get_random_collections( self.cluster.buckets, req_num=1, consider_scopes="all", consider_buckets="all") bucket = BucketUtils.get_bucket_obj(self.cluster.buckets, bucket_dict.keys()[0]) scope_name = bucket_dict[bucket.name]["scopes"].keys()[0] collection_name = bucket_dict[bucket.name][ "scopes"][scope_name]["collections"].keys()[0] scope = BucketUtils.get_scope_obj( bucket, scope_name) collection = BucketUtils.get_collection_obj( scope, collection_name) self.start_doc_loading_tasks(target_vbuckets, scope_name, collection) task_info = dict() task_info[self.doc_loading_task] = \ self.bucket_util.get_doc_op_info_dict( def_bucket, DocLoading.Bucket.DocOps.CREATE, 0, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout=self.sdk_timeout, time_unit="seconds", retry_exceptions=retry_exceptions) self.sleep(10, "Wait for doc_ops to start") self.log.info("Killing {0}:{1} on node {2}" .format(self.process_name, self.service_name, target_node.ip)) remote.kill_process(self.process_name, self.service_name, signum=signum[self.sig_type]) remote.disconnect() # Wait for tasks completion and validate failures if self.transaction_load_task: self.task.jython_task_manager.get_task_result( self.transaction_load_task) if self.N1qltxn: self.task.jython_task_manager.get_task_result( self.N1ql_load_task) self.task_manager.get_task_result(self.doc_loading_task) self.bucket_util.verify_doc_op_task_exceptions(task_info, self.cluster) self.bucket_util.log_doc_ops_task_failures(task_info) # Verification stats verification_dict = dict() verification_dict["ops_create"] = 2*self.num_items verification_dict["sync_write_aborted_count"] = 0 verification_dict["rollback_item_count"] = 0 verification_dict["pending_writes"] = 0 if self.durability_level: verification_dict["sync_write_committed_count"] = 2*self.num_items if self.bucket_type == Bucket.Type.EPHEMERAL \ and self.process_name == "memcached": result = self.task.rebalance(self.servers[:self.nodes_init], [], []) self.assertTrue(result, "Rebalance failed") # Validate doc count if self.process_name != "memcached": stats_failed = \ self.durability_helper.verify_vbucket_details_stats( def_bucket, self.cluster_util.get_kv_nodes(), vbuckets=self.cluster_util.vbuckets, expected_val=verification_dict) if stats_failed: self.fail("Cbstats verification failed") # Doc count validation per collection if not self.N1qltxn and self.atomicity is False: self.bucket_util.validate_docs_per_collections_all_buckets( self.cluster)
def test_stop_process(self): """ 1. Starting loading docs into the default bucket 2. Stop the requested process, which will impact the memcached operations 3. Wait for load bucket task to complete 4. Validate the docs for durability """ error_to_simulate = self.input.param("simulate_error", None) target_node = self.getTargetNode() remote = RemoteMachineShellConnection(target_node) error_sim = CouchbaseError(self.log, remote) target_vbuckets = CrashTest.getVbucketNumbers( remote, self.bucket.name, self.target_node) bucket_dict = BucketUtils.get_random_collections( self.cluster.buckets, req_num=1, consider_scopes="all", consider_buckets="all") bucket = BucketUtils.get_bucket_obj(self.cluster.buckets, bucket_dict.keys()[0]) scope_name = bucket_dict[bucket.name]["scopes"].keys()[0] collection_name = bucket_dict[bucket.name][ "scopes"][scope_name]["collections"].keys()[0] scope = BucketUtils.get_scope_obj( bucket, scope_name) collection = BucketUtils.get_collection_obj(scope, collection_name) if len(target_vbuckets) == 0: self.log.error("No target vbucket list generated to load data") remote.disconnect() return self.start_doc_loading_tasks(target_vbuckets, scope_name, collection) # Induce the error condition error_sim.create(error_to_simulate) self.sleep(20, "Wait before reverting the error condition") # Revert the simulated error condition and close the ssh session error_sim.revert(error_to_simulate) remote.disconnect() # Wait for doc loading task to complete self.task.jython_task_manager.get_task_result(self.doc_loading_task) if self.atomicity: self.task.jython_task_manager.get_task_result( self.transaction_load_task) elif self.N1qltxn: self.task.jython_task_manager.get_task_result( self.N1ql_load_task) if len(self.doc_loading_task.fail.keys()) != 0: if self.target_node == "active" or self.num_replicas in [2, 3]: self.log_failure("Unwanted failures for keys: %s" % self.doc_loading_task.fail.keys()) validate_passed = \ self.durability_helper.validate_durability_exception( self.doc_loading_task.fail, SDKException.DurabilityAmbiguousException) if not validate_passed: self.log_failure("Unwanted exception seen during validation") # Get SDK client for CRUD retries sdk_client = self.sdk_client_pool.get_client_for_bucket(self.bucket) for doc_key, crud_result in self.doc_loading_task.fail.items(): result = sdk_client.crud(DocLoading.Bucket.DocOps.CREATE, doc_key, crud_result["value"], replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout=self.sdk_timeout) if result["status"] is False: self.log_failure("Retry of doc_key %s failed: %s" % (doc_key, result["error"])) # Close the SDK connection self.sdk_client_pool.release_client(sdk_client) self.validate_test_failure() self.bucket_util._wait_for_stats_all_buckets(self.cluster.buckets) # Update self.num_items and validate docs per collection if not self.N1qltxn and self.atomicity is False: self.bucket_util.validate_docs_per_collections_all_buckets( self.cluster)