def test_crash_process(self):
"""
1. Starting loading docs into the default bucket
2. Crash the requested process, which will not impact the
memcached operations
3. Wait for load bucket task to complete
4. Validate the docs for durability
"""
def_bucket = self.cluster.buckets[0]
target_node = self.getTargetNode()
remote = RemoteMachineShellConnection(target_node)
target_vbuckets = range(0, self.cluster_util.vbuckets)
retry_exceptions = list()
self.transaction_load_task = None
self.doc_loading_task = None
self.N1ql_load_task = None
# If Memcached is killed, we should not perform KV ops on
# particular node. If not we can target all nodes for KV operation.
if self.process_name == "memcached":
target_vbuckets = CrashTest.getVbucketNumbers(
remote, def_bucket.name, self.target_node)
if self.target_node == "active":
retry_exceptions = [SDKException.TimeoutException]
if len(target_vbuckets) == 0:
self.log.error("No target vbucket list generated to load data")
remote.disconnect()
return
bucket_dict = BucketUtils.get_random_collections(
self.cluster.buckets,
req_num=1,
consider_scopes="all",
consider_buckets="all")
bucket = BucketUtils.get_bucket_obj(self.cluster.buckets,
bucket_dict.keys()[0])
scope_name = bucket_dict[bucket.name]["scopes"].keys()[0]
collection_name = bucket_dict[bucket.name][
"scopes"][scope_name]["collections"].keys()[0]
scope = BucketUtils.get_scope_obj(
bucket, scope_name)
collection = BucketUtils.get_collection_obj(
scope, collection_name)
self.start_doc_loading_tasks(target_vbuckets, scope_name, collection)
task_info = dict()
task_info[self.doc_loading_task] = \
self.bucket_util.get_doc_op_info_dict(
def_bucket, DocLoading.Bucket.DocOps.CREATE, 0,
replicate_to=self.replicate_to, persist_to=self.persist_to,
durability=self.durability_level,
timeout=self.sdk_timeout, time_unit="seconds",
retry_exceptions=retry_exceptions)
self.sleep(10, "Wait for doc_ops to start")
self.log.info("Killing {0}:{1} on node {2}"
.format(self.process_name, self.service_name,
target_node.ip))
remote.kill_process(self.process_name, self.service_name,
signum=signum[self.sig_type])
remote.disconnect()
# Wait for tasks completion and validate failures
if self.transaction_load_task:
self.task.jython_task_manager.get_task_result(
self.transaction_load_task)
if self.N1qltxn:
self.task.jython_task_manager.get_task_result(
self.N1ql_load_task)
self.task_manager.get_task_result(self.doc_loading_task)
self.bucket_util.verify_doc_op_task_exceptions(task_info,
self.cluster)
self.bucket_util.log_doc_ops_task_failures(task_info)
# Verification stats
verification_dict = dict()
verification_dict["ops_create"] = 2*self.num_items
verification_dict["sync_write_aborted_count"] = 0
verification_dict["rollback_item_count"] = 0
verification_dict["pending_writes"] = 0
if self.durability_level:
verification_dict["sync_write_committed_count"] = 2*self.num_items
if self.bucket_type == Bucket.Type.EPHEMERAL \
and self.process_name == "memcached":
result = self.task.rebalance(self.servers[:self.nodes_init],
[], [])
self.assertTrue(result, "Rebalance failed")
# Validate doc count
if self.process_name != "memcached":
stats_failed = \
self.durability_helper.verify_vbucket_details_stats(
def_bucket, self.cluster_util.get_kv_nodes(),
vbuckets=self.cluster_util.vbuckets,
expected_val=verification_dict)
if stats_failed:
self.fail("Cbstats verification failed")
# Doc count validation per collection
if not self.N1qltxn and self.atomicity is False:
self.bucket_util.validate_docs_per_collections_all_buckets(
self.cluster)
def test_stop_process(self):
"""
1. Starting loading docs into the default bucket
2. Stop the requested process, which will impact the
memcached operations
3. Wait for load bucket task to complete
4. Validate the docs for durability
"""
error_to_simulate = self.input.param("simulate_error", None)
target_node = self.getTargetNode()
remote = RemoteMachineShellConnection(target_node)
error_sim = CouchbaseError(self.log, remote)
target_vbuckets = CrashTest.getVbucketNumbers(
remote, self.bucket.name, self.target_node)
bucket_dict = BucketUtils.get_random_collections(
self.cluster.buckets,
req_num=1,
consider_scopes="all",
consider_buckets="all")
bucket = BucketUtils.get_bucket_obj(self.cluster.buckets,
bucket_dict.keys()[0])
scope_name = bucket_dict[bucket.name]["scopes"].keys()[0]
collection_name = bucket_dict[bucket.name][
"scopes"][scope_name]["collections"].keys()[0]
scope = BucketUtils.get_scope_obj(
bucket, scope_name)
collection = BucketUtils.get_collection_obj(scope, collection_name)
if len(target_vbuckets) == 0:
self.log.error("No target vbucket list generated to load data")
remote.disconnect()
return
self.start_doc_loading_tasks(target_vbuckets, scope_name, collection)
# Induce the error condition
error_sim.create(error_to_simulate)
self.sleep(20, "Wait before reverting the error condition")
# Revert the simulated error condition and close the ssh session
error_sim.revert(error_to_simulate)
remote.disconnect()
# Wait for doc loading task to complete
self.task.jython_task_manager.get_task_result(self.doc_loading_task)
if self.atomicity:
self.task.jython_task_manager.get_task_result(
self.transaction_load_task)
elif self.N1qltxn:
self.task.jython_task_manager.get_task_result(
self.N1ql_load_task)
if len(self.doc_loading_task.fail.keys()) != 0:
if self.target_node == "active" or self.num_replicas in [2, 3]:
self.log_failure("Unwanted failures for keys: %s"
% self.doc_loading_task.fail.keys())
validate_passed = \
self.durability_helper.validate_durability_exception(
self.doc_loading_task.fail,
SDKException.DurabilityAmbiguousException)
if not validate_passed:
self.log_failure("Unwanted exception seen during validation")
# Get SDK client for CRUD retries
sdk_client = self.sdk_client_pool.get_client_for_bucket(self.bucket)
for doc_key, crud_result in self.doc_loading_task.fail.items():
result = sdk_client.crud(DocLoading.Bucket.DocOps.CREATE,
doc_key,
crud_result["value"],
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout=self.sdk_timeout)
if result["status"] is False:
self.log_failure("Retry of doc_key %s failed: %s"
% (doc_key, result["error"]))
# Close the SDK connection
self.sdk_client_pool.release_client(sdk_client)
self.validate_test_failure()
self.bucket_util._wait_for_stats_all_buckets(self.cluster.buckets)
# Update self.num_items and validate docs per collection
if not self.N1qltxn and self.atomicity is False:
self.bucket_util.validate_docs_per_collections_all_buckets(
self.cluster)
