def enable_error_scenario_and_test_durability(self):
"""
1. Select nodes from the cluster to simulate the specified error
2. Perform CRUD on the target bucket with given timeout
3. Using cbstats to verify the operation succeeds
4. Validate all mutations met the durability condition
"""
error_sim = dict()
shell_conn = dict()
cbstat_obj = dict()
failover_info = dict()
vb_info_info = dict()
target_vbuckets = range(0, self.vbuckets)
active_vbs_in_target_nodes = list()
failover_info["init"] = dict()
failover_info["afterCrud"] = dict()
vb_info_info["init"] = dict()
vb_info_info["afterCrud"] = dict()
disk_related_errors = [DiskError.DISK_FULL,
DiskError.FAILOVER_DISK,
"stop_persistence"]
self.log.info("Selecting nodes to simulate error condition")
target_nodes = self.getTargetNodes()
self.log.info("Will simulate error condition on %s" % target_nodes)
for node in target_nodes:
shell_conn[node.ip] = RemoteMachineShellConnection(node)
cbstat_obj[node.ip] = Cbstats(shell_conn[node.ip])
active_vbs = cbstat_obj[node.ip] .vbucket_list(self.bucket.name,
"active")
active_vbs_in_target_nodes += active_vbs
vb_info_info["init"][node.ip] = cbstat_obj[node.ip].vbucket_seqno(
self.bucket.name)
failover_info["init"][node.ip] = \
cbstat_obj[node.ip].failover_stats(self.bucket.name)
if self.simulate_error \
in [DiskError.DISK_FULL, DiskError.FAILOVER_DISK]:
error_sim = DiskError(self.log, self.task_manager,
self.cluster.master, target_nodes,
60, 0, False, 120,
disk_location="/data")
error_sim.create(action=self.simulate_error)
else:
for node in target_nodes:
# Create shell_connections
shell_conn[node.ip] = RemoteMachineShellConnection(node)
# Perform specified action
error_sim[node.ip] = CouchbaseError(self.log,
shell_conn[node.ip])
error_sim[node.ip].create(self.simulate_error,
bucket_name=self.bucket.name)
if self.simulate_error not in disk_related_errors:
# Remove active vbuckets from doc_loading to avoid errors
target_vbuckets = list(set(target_vbuckets)
^ set(active_vbs_in_target_nodes))
# Perform CRUDs with induced error scenario is active
tasks = list()
gen_create = doc_generator(self.key, self.num_items,
self.num_items+self.crud_batch_size,
target_vbucket=target_vbuckets)
gen_delete = doc_generator(self.key, 0,
int(self.num_items/3),
target_vbucket=target_vbuckets)
gen_read = doc_generator(self.key, int(self.num_items/3),
self.num_items,
target_vbucket=target_vbuckets)
gen_update = doc_generator(self.key, int(self.num_items/2),
self.num_items,
target_vbucket=target_vbuckets)
self.log.info("Starting parallel doc_ops - Create/Read/Update/Delete")
tasks.append(self.task.async_load_gen_docs(
self.cluster, self.bucket, gen_create, "create", 0,
batch_size=10, process_concurrency=1,
replicate_to=self.replicate_to, persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout, retries=self.sdk_retries))
tasks.append(self.task.async_load_gen_docs(
self.cluster, self.bucket, gen_update, "update", 0,
batch_size=10, process_concurrency=1,
replicate_to=self.replicate_to, persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout, retries=self.sdk_retries))
tasks.append(self.task.async_load_gen_docs(
self.cluster, self.bucket, gen_read, "read", 0,
batch_size=10, process_concurrency=1,
replicate_to=self.replicate_to, persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout, retries=self.sdk_retries))
tasks.append(self.task.async_load_gen_docs(
self.cluster, self.bucket, gen_delete, "delete", 0,
batch_size=10, process_concurrency=1,
replicate_to=self.replicate_to, persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout, retries=self.sdk_retries))
# Update num_items value accordingly to the CRUD performed
self.num_items += len(gen_create.doc_keys) - len(gen_delete.doc_keys)
# Wait for document_loader tasks to complete
for task in tasks:
self.task.jython_task_manager.get_task_result(task)
# Verify there is not failed docs in the task
if len(task.fail.keys()) != 0:
self.log_failure("Some CRUD failed for {0}".format(task.fail))
if self.simulate_error \
not in [DiskError.DISK_FULL, DiskError.FAILOVER_DISK]:
# Revert the induced error condition
for node in target_nodes:
error_sim[node.ip].revert(self.simulate_error,
bucket_name=self.bucket.name)
# Disconnect the shell connection
shell_conn[node.ip].disconnect()
# Create a SDK client connection to retry operation
client = SDKClient(RestConnection(self.cluster.master),
self.bucket.name)
# Retry failed docs (if any)
for index, task in enumerate(tasks):
if index == 0:
op_type = "create"
elif index == 1:
op_type = "update"
elif index == 2:
op_type = "read"
elif index == 3:
op_type = "delete"
op_failed = self.durability_helper.retry_with_no_error(
client, task.fail, op_type)
if op_failed:
self.log_failure(
"CRUD '{0}' failed on retry with no error condition"
.format(op_type))
# Close the SDK connection
client.close()
# Fetch latest failover stats and validate the values are updated
self.log.info("Validating failover and seqno cbstats")
for node in target_nodes:
vb_info_info["afterCrud"][node.ip] = \
cbstat_obj[node.ip].vbucket_seqno(self.bucket.name)
failover_info["afterCrud"][node.ip] = \
cbstat_obj[node.ip].failover_stats(self.bucket.name)
# Failover validation
if self.simulate_error in disk_related_errors:
val = failover_info["init"][node.ip] \
== failover_info["afterCrud"][node.ip]
error_msg = "Failover stats got updated"
else:
val = failover_info["init"][node.ip] \
!= failover_info["afterCrud"][node.ip]
error_msg = "Failover stats not updated after error condition"
self.assertTrue(val, msg=error_msg)
# Seq_no validation (High level)
val = vb_info_info["init"][node.ip] \
!= vb_info_info["afterCrud"][node.ip]
self.assertTrue(val, msg="vbucket seq_no not updated after CRUDs")
# Verify doc count
self.log.info("Validating doc count")
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
self.validate_test_failure()
def test_with_persistence_issues(self):
"""
Test to make sure timeout is handled in durability calls
and document CRUDs are successful even with disk related failures
1. Select nodes from the cluster to simulate the specified error
2. Perform CRUD on the target bucket with given timeout
3. Using cbstats to verify the operation succeeds
4. Validate all mutations are succeeded
Note: self.sdk_timeout value is considered as 'seconds'
"""
if self.durability_level in [
Bucket.DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE,
Bucket.DurabilityLevel.PERSIST_TO_MAJORITY
]:
self.log.critical("Test not valid for persistence durability")
return
error_sim = dict()
shell_conn = dict()
cbstat_obj = dict()
failover_info = dict()
vb_info_info = dict()
active_vbs_in_target_nodes = list()
failover_info["init"] = dict()
failover_info["afterCrud"] = dict()
vb_info_info["init"] = dict()
vb_info_info["afterCrud"] = dict()
self.log.info("Selecting nodes to simulate error condition")
target_nodes = DurabilityHelper.getTargetNodes(self.cluster,
self.nodes_init,
self.num_nodes_affected)
self.log.info("Simulate error condition on %s" % target_nodes)
for node in target_nodes:
shell_conn[node.ip] = RemoteMachineShellConnection(node)
cbstat_obj[node.ip] = Cbstats(shell_conn[node.ip])
active_vbs_in_target_nodes += cbstat_obj[node.ip].vbucket_list(
self.bucket.name, "active")
vb_info_info["init"][node.ip] = cbstat_obj[node.ip].vbucket_seqno(
self.bucket.name)
failover_info["init"][node.ip] = \
cbstat_obj[node.ip].failover_stats(self.bucket.name)
if self.simulate_error \
in [DiskError.DISK_FULL, DiskError.DISK_FAILURE]:
error_sim = DiskError(self.log,
self.task_manager,
self.cluster.master,
target_nodes,
60,
0,
False,
120,
disk_location="/data")
error_sim.create(action=self.simulate_error)
else:
for node in target_nodes:
# Create shell_connections
shell_conn[node.ip] = RemoteMachineShellConnection(node)
# Perform specified action
error_sim[node.ip] = CouchbaseError(self.log,
shell_conn[node.ip])
error_sim[node.ip].create(self.simulate_error,
bucket_name=self.bucket.name)
# Perform CRUDs with induced error scenario is active
load_spec = dict()
load_spec["doc_crud"] = dict()
load_spec["doc_crud"][
MetaCrudParams.DocCrud.CREATE_PERCENTAGE_PER_COLLECTION] = 100
load_spec["doc_crud"][
MetaCrudParams.DocCrud.UPDATE_PERCENTAGE_PER_COLLECTION] = 25
load_spec["doc_crud"][
MetaCrudParams.DocCrud.DELETE_PERCENTAGE_PER_COLLECTION] = 25
load_spec["doc_crud"][
MetaCrudParams.DocCrud.COMMON_DOC_KEY] = "test_collections"
self.log.info("Perform 'create', 'update', 'delete' mutations")
doc_loading_task = \
self.bucket_util.run_scenario_from_spec(
self.task,
self.cluster,
self.bucket_util.buckets,
load_spec,
mutation_num=1,
async_load=True)
# Perform new scope/collection creation during doc ops in parallel
self.__perform_collection_crud(mutation_num=2)
# Wait for doc_loading to complete and validate the doc ops
self.task_manager.get_task_result(doc_loading_task)
self.bucket_util.validate_doc_loading_results(doc_loading_task)
if doc_loading_task.result is False:
self.log_failure("Doc CRUDs failed with persistence issue")
if self.simulate_error \
in [DiskError.DISK_FULL, DiskError.DISK_FAILURE]:
error_sim.revert(self.simulate_error)
else:
# Revert the induced error condition
for node in target_nodes:
error_sim[node.ip].revert(self.simulate_error,
bucket_name=self.bucket.name)
# Disconnect the shell connection
shell_conn[node.ip].disconnect()
self.sleep(10, "Wait for node recovery to complete")
# Doc count validation
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.validate_docs_per_collections_all_buckets()
# Fetch latest failover stats and validate the values are updated
self.log.info("Validating failover and seqno cbstats")
for node in target_nodes:
vb_info_info["afterCrud"][node.ip] = \
cbstat_obj[node.ip].vbucket_seqno(self.bucket.name)
failover_info["afterCrud"][node.ip] = \
cbstat_obj[node.ip].failover_stats(self.bucket.name)
# Failover validation
val = \
failover_info["init"][node.ip] \
== failover_info["afterCrud"][node.ip]
error_msg = "Failover stats got updated"
self.assertTrue(val, msg=error_msg)
# Seq_no validation (High level)
val = \
vb_info_info["init"][node.ip] \
!= vb_info_info["afterCrud"][node.ip]
self.assertTrue(val, msg="vbucket seq_no not updated after CRUDs")
self.validate_test_failure()
# Doc count validation
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.validate_docs_per_collections_all_buckets()
def test_with_persistence_issues(self):
"""
Test to make sure timeout is handled in durability calls
and document CRUDs are successful even with disk related failures
1. Select nodes from the cluster to simulate the specified error
2. Perform CRUD on the target bucket with given timeout
3. Using cbstats to verify the operation succeeds
4. Validate all mutations are succeeded
Note: self.sdk_timeout value is considered as 'seconds'
"""
if self.durability_level.upper() in [
Bucket.DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE,
Bucket.DurabilityLevel.PERSIST_TO_MAJORITY]:
self.log.critical("Test not valid for persistence durability")
return
error_sim = dict()
shell_conn = dict()
cbstat_obj = dict()
failover_info = dict()
vb_info_info = dict()
active_vbs_in_target_nodes = list()
failover_info["init"] = dict()
failover_info["afterCrud"] = dict()
vb_info_info["init"] = dict()
vb_info_info["afterCrud"] = dict()
self.log.info("Selecting nodes to simulate error condition")
target_nodes = self.getTargetNodes()
self.log.info("Will simulate error condition on %s" % target_nodes)
for node in target_nodes:
shell_conn[node.ip] = RemoteMachineShellConnection(node)
cbstat_obj[node.ip] = Cbstats(shell_conn[node.ip])
active_vbs_in_target_nodes += cbstat_obj[node.ip].vbucket_list(
self.bucket.name,
"active")
vb_info_info["init"][node.ip] = cbstat_obj[node.ip].vbucket_seqno(
self.bucket.name)
failover_info["init"][node.ip] = \
cbstat_obj[node.ip].failover_stats(self.bucket.name)
if self.simulate_error \
in [DiskError.DISK_FULL, DiskError.DISK_FAILURE]:
error_sim = DiskError(self.log, self.task_manager,
self.cluster.master, target_nodes,
60, 0, False, 120,
disk_location="/data")
error_sim.create(action=self.simulate_error)
else:
for node in target_nodes:
# Create shell_connections
shell_conn[node.ip] = RemoteMachineShellConnection(node)
# Perform specified action
error_sim[node.ip] = CouchbaseError(self.log,
shell_conn[node.ip])
error_sim[node.ip].create(self.simulate_error,
bucket_name=self.bucket.name)
# Perform CRUDs with induced error scenario is active
tasks = list()
gen_create = doc_generator(self.key, self.num_items,
self.num_items+self.crud_batch_size)
gen_delete = doc_generator(self.key, 0,
int(self.num_items/3))
gen_update = doc_generator(self.key, int(self.num_items/2),
self.num_items)
self.log.info("Starting parallel doc_ops - Create/Read/Update/Delete")
tasks.append(self.task.async_load_gen_docs(
self.cluster, self.bucket, gen_create, "create", 0,
batch_size=10, process_concurrency=1,
replicate_to=self.replicate_to, persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout))
tasks.append(self.task.async_load_gen_docs(
self.cluster, self.bucket, gen_update, "update", 0,
batch_size=10, process_concurrency=1,
replicate_to=self.replicate_to, persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout))
# Wait for document_loader tasks to complete
for task in tasks:
self.task.jython_task_manager.get_task_result(task)
# Verify there is not failed docs in the task
if len(task.fail.keys()) != 0:
self.log_failure("Some CRUD failed during {0}: {1}"
.format(task.op_type, task.fail))
tasks.append(self.task.async_load_gen_docs(
self.cluster, self.bucket, gen_update, "read", 0,
batch_size=10, process_concurrency=1,
replicate_to=self.replicate_to, persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout))
tasks.append(self.task.async_load_gen_docs(
self.cluster, self.bucket, gen_delete, "delete", 0,
batch_size=10, process_concurrency=1,
replicate_to=self.replicate_to, persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout))
# Wait for document_loader tasks to complete
for task in tasks[2:]:
self.task.jython_task_manager.get_task_result(task)
# Verify there is not failed docs in the task
if len(task.fail.keys()) != 0:
self.log_failure("Some CRUD failed during {0}: {1}"
.format(task.op_type, task.fail))
if self.simulate_error \
in [DiskError.DISK_FULL, DiskError.DISK_FAILURE]:
error_sim.revert(self.simulate_error)
else:
# Revert the induced error condition
for node in target_nodes:
error_sim[node.ip].revert(self.simulate_error,
bucket_name=self.bucket.name)
# Disconnect the shell connection
shell_conn[node.ip].disconnect()
self.sleep(10, "Wait for node recovery to complete")
# Update num_items value accordingly to the CRUD performed
self.num_items += self.crud_batch_size - int(self.num_items/3)
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
# Create a SDK client connection to retry operation
client = SDKClient([self.cluster.master], self.bucket)
# Retry failed docs (if any)
for index, task in enumerate(tasks):
if index == 0:
op_type = "create"
elif index == 1:
op_type = "update"
elif index == 2:
op_type = "read"
elif index == 3:
op_type = "delete"
op_failed = self.durability_helper.retry_with_no_error(
client, task.fail, op_type)
if op_failed:
self.log_failure(
"CRUD '{0}' failed on retry with no error condition"
.format(op_type))
# Close the SDK connection
client.close()
# Fetch latest failover stats and validate the values are updated
self.log.info("Validating failover and seqno cbstats")
for node in target_nodes:
vb_info_info["afterCrud"][node.ip] = \
cbstat_obj[node.ip].vbucket_seqno(self.bucket.name)
failover_info["afterCrud"][node.ip] = \
cbstat_obj[node.ip].failover_stats(self.bucket.name)
# Failover validation
val = failover_info["init"][node.ip] \
== failover_info["afterCrud"][node.ip]
error_msg = "Failover stats got updated"
self.assertTrue(val, msg=error_msg)
# Seq_no validation (High level)
val = vb_info_info["init"][node.ip] \
!= vb_info_info["afterCrud"][node.ip]
self.assertTrue(val, msg="vbucket seq_no not updated after CRUDs")
# Verify doc count
self.log.info("Validating doc count")
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
self.validate_test_failure()