def test_replica_update(self):
if self.atomicity:
replica_count = 3
else:
replica_count = 4
if self.nodes_init < 2:
self.log.error("Test not supported for < 2 node cluster")
return
doc_ops = self.input.param("doc_ops", "")
bucket_helper = BucketHelper(self.cluster.master)
doc_count = self.num_items
start_doc_for_insert = self.num_items
self.is_sync_write_enabled = DurabilityHelper.is_sync_write_enabled(
self.bucket_durability_level, self.durability_level)
# Replica increment tests
doc_count, start_doc_for_insert = self.generic_replica_update(
doc_count,
doc_ops,
bucket_helper,
range(1, min(replica_count, self.nodes_init)),
start_doc_for_insert)
# Replica decrement tests
_, _ = self.generic_replica_update(
doc_count,
doc_ops,
bucket_helper,
range(min(replica_count, self.nodes_init)-2, -1, -1),
start_doc_for_insert)
def test_rebalance_out_durabilitybreaks_rebalance_in(self):
self.assertTrue(self.num_replicas >= 1,
"Need at-least one replica to run this test")
durability_will_fail = False
def_bucket = self.cluster.buckets[0]
nodes_in_cluster = self.nodes_init
nodes_required_for_durability = int(
floor((self.num_replicas + 1) / 2) + 1)
gen_update = doc_generator(self.key, 0, self.num_items)
num_of_docs_to_insert = 1000
# Rebalance all nodes expect master one-by-one
for _ in range(self.nodes_init - 1):
rebalance_result = self.task.rebalance(
self.cluster.servers[:nodes_in_cluster], [],
[self.cluster.servers[nodes_in_cluster - 1]])
self.assertTrue(rebalance_result)
nodes_in_cluster -= 1
if nodes_in_cluster < nodes_required_for_durability:
if DurabilityHelper.is_sync_write_enabled(
self.bucket_durability_level, self.durability_level):
durability_will_fail = True
tasks = list()
# Perform CRUD operations after rebalance_out to verify the
# durability outcome
gen_create = doc_generator(self.key, self.num_items,
self.num_items + num_of_docs_to_insert)
if self.atomicity:
tasks.append(
self.task.async_load_gen_docs_atomicity(
self.cluster,
self.cluster.buckets,
gen_create,
"rebalance_update",
exp=0,
batch_size=10,
process_concurrency=self.process_concurrency,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,
update_count=self.update_count,
transaction_timeout=self.transaction_timeout,
commit=True,
durability=self.durability_level,
sync=True,
num_threads=1,
record_fail=True,
defer=self.defer))
self.num_items += num_of_docs_to_insert
def_bucket.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items += num_of_docs_to_insert
else:
tasks.append(
self.task.async_load_gen_docs(
self.cluster,
def_bucket,
gen_create,
"create",
exp=0,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
batch_size=20,
process_concurrency=4,
sdk_client_pool=self.sdk_client_pool))
tasks.append(
self.task.async_load_gen_docs(
self.cluster,
def_bucket,
gen_update,
"update",
0,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
batch_size=20,
process_concurrency=4,
sdk_client_pool=self.sdk_client_pool))
# Wait for all CRUD tasks to complete
for task in tasks:
self.task.jython_task_manager.get_task_result(task)
if durability_will_fail:
self.log.info(
"Durability broken with cluster {0}, replica {1}".format(
self.cluster.servers[:nodes_in_cluster],
self.num_replicas))
assert_msg = \
"Items crud succeeded although majority condition failed."
if not self.atomicity:
for task in tasks:
if task.op_type == "create":
self.assertTrue(
len(task.fail) == num_of_docs_to_insert,
assert_msg)
elif task.op_type == "update":
self.assertTrue(
len(task.fail) == (gen_update.end -
gen_update.start),
assert_msg)
break
elif not self.atomicity:
# If durability works fine, re-calculate the self.num_items
self.num_items += num_of_docs_to_insert
def_bucket.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items += num_of_docs_to_insert
# Reset the tasks list to reuse
tasks = list()
# Create tasks for doc verification
tasks.append(
self.task.async_validate_docs(
self.cluster,
def_bucket,
gen_update,
"update",
0,
batch_size=10,
process_concurrency=4,
sdk_client_pool=self.sdk_client_pool))
tasks.append(
self.task.async_validate_docs(
self.cluster,
def_bucket,
gen_create,
"create",
0,
batch_size=10,
process_concurrency=4,
sdk_client_pool=self.sdk_client_pool))
# Wait for all verification tasks to complete
for task in tasks:
self.task.jython_task_manager.get_task_result(task)
# Rebalance-in single node back into the cluster
rebalance_result = self.task.rebalance(
self.cluster.servers[:nodes_in_cluster],
self.cluster.servers[nodes_in_cluster:nodes_in_cluster + 1], [])
# Wait for rebalance-in task to complete
self.assertTrue(rebalance_result)
# Reset the tasks list
tasks = list()
# Perform CRUD operations after rebalance_in to verify the
# durability is working as expected again
gen_create = doc_generator(self.key, self.num_items,
self.num_items + num_of_docs_to_insert)
if self.atomicity:
tasks.append(
self.task.async_load_gen_docs_atomicity(
self.cluster,
self.cluster.buckets,
gen_create,
"rebalance_update",
exp=0,
batch_size=10,
process_concurrency=self.process_concurrency,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,
update_count=self.update_count,
transaction_timeout=self.transaction_timeout,
commit=True,
durability=self.durability_level,
sync=True,
num_threads=1,
record_fail=True,
defer=self.defer))
else:
tasks.append(
self.task.async_load_gen_docs(
self.cluster,
def_bucket,
gen_create,
"create",
exp=0,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
batch_size=20,
sdk_client_pool=self.sdk_client_pool))
tasks.append(
self.task.async_load_gen_docs(
self.cluster,
def_bucket,
gen_update,
"update",
0,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
batch_size=20,
sdk_client_pool=self.sdk_client_pool))
for task in tasks:
self.task.jython_task_manager.get_task_result(task)
# Wait for all CRUD tasks to complete
if not self.atomicity:
assert_msg = \
"Items crud succeeded although majority condition failed."
for task in tasks:
if task.op_type == "create":
self.assertTrue(len(task.fail) == 0, assert_msg)
elif task.op_type == "update":
self.assertTrue(len(task.fail) == 0, assert_msg)
# After rebalance-in durability should work fine
# So recalculating the self.num_items to match gen_create loader
self.num_items += num_of_docs_to_insert
def_bucket.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items += num_of_docs_to_insert
# Reset the tasks list
tasks = list()
# Create tasks for doc verification
tasks.append(
self.task.async_validate_docs(
self.cluster,
def_bucket,
gen_update,
"update",
0,
batch_size=10,
process_concurrency=4,
sdk_client_pool=self.sdk_client_pool))
tasks.append(
self.task.async_validate_docs(
self.cluster,
def_bucket,
gen_create,
"create",
0,
batch_size=10,
process_concurrency=4,
sdk_client_pool=self.sdk_client_pool))
# Wait for all doc verification tasks to complete
for task in tasks:
self.task.jython_task_manager.get_task_result(task)
# Verify doc load count to match the overall CRUDs
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
self.bucket_util.validate_docs_per_collections_all_buckets(
self.cluster)
def test_collection_not_exists(self):
"""
1. Load docs into required collection
2. Validate docs based on the targeted collection
3. Create non-default scope/collection for CRUDs to happen
4. Perform doc_ops again and perform CRUDs
5. Drop the target collection and validate the CollectionNotExists
exception from client side
6. Recreate non-default collection and re-create the docs and validate
"""
def validate_vb_detail_stats():
failed = durability_helper.verify_vbucket_details_stats(
self.bucket,
self.cluster_util.get_kv_nodes(self.cluster),
vbuckets=self.cluster.vbuckets,
expected_val=verification_dict)
if failed:
self.log_failure("vBucket_details validation failed")
self.bucket_util.validate_docs_per_collections_all_buckets(
self.cluster)
sync_write_enabled = DurabilityHelper.is_sync_write_enabled(
self.bucket_durability_level, self.durability_level)
num_cols_in_bucket = 0
for _, scope in self.bucket.scopes.items():
for _, _ in scope.collections.items():
num_cols_in_bucket += 1
verification_dict = dict()
verification_dict["ops_create"] = num_cols_in_bucket * self.num_items
verification_dict["ops_update"] = 0
verification_dict["ops_delete"] = 0
verification_dict["rollback_item_count"] = 0
verification_dict["sync_write_aborted_count"] = 0
verification_dict["sync_write_committed_count"] = 0
durability_helper = DurabilityHelper(self.log,
len(self.cluster.kv_nodes),
durability=self.durability_level)
drop_scope = self.input.param("drop_scope", False)
if self.scope_name != CbServer.default_scope:
self.scope_name = self.bucket_util.get_random_name()
if self.collection_name != CbServer.default_collection:
self.collection_name = self.bucket_util.get_random_name()
# Doc generator used for mutations
doc_gen = doc_generator("test_col_not_exists", 0, 10)
# Acquire SDK client for mutations
client = self.sdk_client_pool.get_client_for_bucket(
self.bucket, self.scope_name, self.collection_name)
doc_ttl, _ = \
SDKExceptionTests.__get_random_doc_ttl_and_durability_level()
self.log.info(
"Creating docs with doc_ttl %s into %s:%s:%s" %
(doc_ttl, self.bucket.name, self.scope_name, self.collection_name))
retry_reason = SDKException.RetryReason
while doc_gen.has_next():
key, value = doc_gen.next()
result = client.crud("create",
key,
value,
exp=doc_ttl,
durability=self.durability_level,
timeout=30)
if self.collection_name == CbServer.default_collection:
if result["status"] is False:
self.log_failure("Create doc failed for key: %s" % key)
else:
verification_dict["ops_create"] += 1
if sync_write_enabled:
verification_dict["sync_write_committed_count"] += 1
self.bucket.scopes[self.scope_name].collections[
self.collection_name].num_items += 1
elif result["status"] is True:
self.log_failure("Create didn't fail as expected for key: %s" %
key)
elif (SDKException.AmbiguousTimeoutException
not in str(result["error"])
or retry_reason.COLLECTION_NOT_FOUND
not in str(result["error"])) \
and (
SDKException.RequestCanceledException
not in str(result["error"])
or retry_reason.COLLECTION_MAP_REFRESH_IN_PROGRESS
not in str(result["error"])):
self.log_failure("Invalid exception for key %s: %s" %
(key, result["error"]))
validate_vb_detail_stats()
# Create required scope/collection for successful CRUD operation
self.create_scope_collection()
# Reset doc_gen itr value for retry purpose
doc_gen.reset()
doc_ttl, _ = \
SDKExceptionTests.__get_random_doc_ttl_and_durability_level()
self.log.info(
"Creating docs with doc_ttl %s into %s:%s:%s" %
(doc_ttl, self.bucket.name, self.scope_name, self.collection_name))
op_type = "create"
if self.collection_name == CbServer.default_collection:
op_type = "update"
while doc_gen.has_next():
key, value = doc_gen.next()
result = client.crud(op_type,
key,
value,
exp=doc_ttl,
durability=self.durability_level)
if result["status"] is False:
self.log_failure("Create fail for key %s: %s" % (key, result))
else:
if op_type == "create":
verification_dict["ops_create"] += 1
self.bucket.scopes[self.scope_name].collections[
self.collection_name].num_items += 1
else:
verification_dict["ops_update"] += 1
if sync_write_enabled:
verification_dict["sync_write_committed_count"] += 1
validate_vb_detail_stats()
self.validate_test_failure()
if drop_scope:
self.log.info("Dropping scope %s" % self.scope_name)
self.bucket_util.drop_scope(self.cluster.master, self.bucket,
self.scope_name)
else:
self.log.info("Dropping collection %s:%s" %
(self.scope_name, self.collection_name))
self.bucket_util.drop_collection(self.cluster.master, self.bucket,
self.scope_name,
self.collection_name)
validate_vb_detail_stats()
self.validate_test_failure()
# Reset doc_gen itr value for retry purpose
doc_gen.reset()
while doc_gen.has_next():
key, value = doc_gen.next()
result = client.crud("create",
key,
value,
exp=doc_ttl,
durability=self.durability_level)
if result["status"] is True:
self.log_failure("Create doc succeeded for dropped collection")
validate_vb_detail_stats()
self.validate_test_failure()
# Re-create the dropped collection
self.create_scope_collection(create_scope=drop_scope)
if self.collection_name != CbServer.default_collection:
doc_gen.reset()
while doc_gen.has_next():
key, value = doc_gen.next()
result = client.crud("create",
key,
value,
exp=doc_ttl,
durability=self.durability_level)
if result["status"] is False:
self.log_failure("Create failed after collection recreate "
"for key %s: %s" % (key, result["error"]))
else:
verification_dict["ops_create"] += 1
if sync_write_enabled:
verification_dict["sync_write_committed_count"] += 1
self.bucket.scopes[self.scope_name].collections[
self.collection_name].num_items += 1
validate_vb_detail_stats()
# Release the acquired client
self.sdk_client_pool.release_client(client)
self.validate_test_failure()
def setUp(self):
super(CrashTest, self).setUp()
self.doc_ops = self.input.param("doc_ops", None)
self.process_name = self.input.param("process", None)
self.service_name = self.input.param("service", "data")
self.sig_type = self.input.param("sig_type", "SIGKILL").upper()
self.target_node = self.input.param("target_node", "active")
self.pre_warmup_stats = {}
self.timeout = 120
self.new_docs_to_add = 10000
if self.doc_ops is not None:
self.doc_ops = self.doc_ops.split(";")
if not self.atomicity:
self.durability_helper = DurabilityHelper(
self.log,
self.nodes_init,
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
self.bucket_util.create_default_bucket(
cluster=self.cluster,
bucket_type=self.bucket_type,
ram_quota=self.bucket_size,
replica=self.num_replicas,
compression_mode="off",
storage=self.bucket_storage,
eviction_policy=self.bucket_eviction_policy)
self.bucket_util.add_rbac_user(self.cluster.master)
if self.sdk_client_pool:
self.log.info("Creating SDK clients for client_pool")
for bucket in self.cluster.buckets:
self.sdk_client_pool.create_clients(
bucket, [self.cluster.master],
self.sdk_pool_capacity,
compression_settings=self.sdk_compression)
self.__is_sync_write_enabled = DurabilityHelper.is_sync_write_enabled(
self.bucket_durability_level, self.durability_level)
verification_dict = dict()
verification_dict["ops_create"] = self.num_items
verification_dict["sync_write_aborted_count"] = 0
verification_dict["rollback_item_count"] = 0
verification_dict["pending_writes"] = 0
if self.__is_sync_write_enabled:
verification_dict["sync_write_committed_count"] = self.num_items
# Load initial documents into the buckets
self.log.info("Loading initial documents")
gen_create = doc_generator(self.key,
0,
self.num_items,
key_size=self.key_size,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster.vbuckets)
if self.atomicity:
task = self.task.async_load_gen_docs_atomicity(
self.cluster,
self.cluster.buckets,
gen_create,
"create",
exp=0,
batch_size=10,
process_concurrency=self.process_concurrency,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
update_count=self.update_count,
transaction_timeout=self.transaction_timeout,
commit=True,
sync=self.sync)
self.task.jython_task_manager.get_task_result(task)
else:
for bucket in self.cluster.buckets:
task = self.task.async_load_gen_docs(
self.cluster,
bucket,
gen_create,
DocLoading.Bucket.DocOps.CREATE,
self.maxttl,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
batch_size=10,
process_concurrency=8,
sdk_client_pool=self.sdk_client_pool)
self.task.jython_task_manager.get_task_result(task)
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
# Verify cbstats vbucket-details
stats_failed = \
self.durability_helper.verify_vbucket_details_stats(
bucket, self.cluster_util.get_kv_nodes(self.cluster),
vbuckets=self.cluster.vbuckets,
expected_val=verification_dict)
if stats_failed:
self.fail("Cbstats verification failed")
self.bucket_util.verify_stats_all_buckets(self.cluster,
self.num_items)
self.cluster_util.print_cluster_stats(self.cluster)
self.bucket_util.print_bucket_stats(self.cluster)
self.log.info("==========Finished CrashTest setup========")
def setUp(self):
super(BasicOps, self).setUp()
self.is_sync_write_enabled = DurabilityHelper.is_sync_write_enabled(
self.bucket_durability_level, self.durability_level)
self.log.info("==========Finished BasisOps base setup========")
def setUp(self):
super(CrashTest, self).setUp()
self.doc_ops = self.input.param("doc_ops", None)
self.process_name = self.input.param("process", None)
self.service_name = self.input.param("service", "data")
self.sig_type = self.input.param("sig_type", "SIGKILL").upper()
self.target_node = self.input.param("target_node", "active")
self.client_type = self.input.param("client_type", "sdk").lower()
self.N1qltxn = self.input.param("N1qltxn", False)
self.allowed_hosts = self.input.param("allowed_hosts", False)
self.pre_warmup_stats = dict()
self.timeout = 120
self.new_docs_to_add = 10000
if self.doc_ops is not None:
self.doc_ops = self.doc_ops.split(";")
if not self.atomicity:
self.durability_helper = DurabilityHelper(
self.log,
self.nodes_init,
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
self.__is_sync_write_enabled = DurabilityHelper.is_sync_write_enabled(
self.bucket_durability_level, self.durability_level)
verification_dict = dict()
verification_dict["ops_create"] = \
self.cluster.buckets[0].scopes[
CbServer.default_scope].collections[
CbServer.default_collection].num_items
verification_dict["sync_write_aborted_count"] = 0
verification_dict["rollback_item_count"] = 0
verification_dict["pending_writes"] = 0
if self.__is_sync_write_enabled:
verification_dict["sync_write_committed_count"] = \
verification_dict["ops_create"]
# Load initial documents into the buckets
transaction_gen_create = doc_generator(
"transaction_key",
0,
self.num_items,
key_size=self.key_size,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster.vbuckets)
gen_create = doc_generator(self.key,
0,
self.num_items,
key_size=self.key_size,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster.vbuckets)
if self.atomicity:
transaction_task = self.task.async_load_gen_docs_atomicity(
self.cluster,
self.cluster.buckets,
transaction_gen_create,
DocLoading.Bucket.DocOps.CREATE,
exp=0,
batch_size=10,
process_concurrency=self.process_concurrency,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
update_count=self.update_count,
transaction_timeout=self.transaction_timeout,
commit=True,
sync=self.sync)
self.task.jython_task_manager.get_task_result(transaction_task)
for bucket in self.cluster.buckets:
task = self.task.async_load_gen_docs(
self.cluster,
bucket,
gen_create,
DocLoading.Bucket.DocOps.CREATE,
self.maxttl,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
batch_size=10,
process_concurrency=8)
self.task.jython_task_manager.get_task_result(task)
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
self.cluster.buckets[0].scopes[CbServer.default_scope].collections[
CbServer.default_collection].num_items += self.num_items
verification_dict["ops_create"] += self.num_items
if self.__is_sync_write_enabled \
and self.durability_level != Bucket.DurabilityLevel.NONE:
verification_dict["sync_write_committed_count"] += \
self.num_items
# Verify cbstats vbucket-details
stats_failed = self.durability_helper.verify_vbucket_details_stats(
bucket,
self.cluster_util.get_kv_nodes(self.cluster),
vbuckets=self.cluster.vbuckets,
expected_val=verification_dict)
if self.atomicity is False:
if stats_failed:
self.fail("Cbstats verification failed")
self.bucket_util.verify_stats_all_buckets(
self.cluster,
self.cluster.buckets[0].scopes[CbServer.default_scope].
collections[CbServer.default_collection].num_items)
self.bucket = self.cluster.buckets[0]
if self.N1qltxn:
self.n1ql_server = self.cluster_util.get_nodes_from_services_map(
cluster=self.cluster,
service_type=CbServer.Services.N1QL,
get_all_nodes=True)
self.n1ql_helper = N1QLHelper(server=self.n1ql_server,
use_rest=True,
buckets=self.cluster.buckets,
log=self.log,
scan_consistency='REQUEST_PLUS',
num_collection=3,
num_buckets=1,
num_savepoints=1,
override_savepoint=False,
num_stmt=10,
load_spec=self.data_spec_name)
self.bucket_col = self.n1ql_helper.get_collections()
self.stmts = self.n1ql_helper.get_stmt(self.bucket_col)
self.stmts = self.n1ql_helper.create_full_stmts(self.stmts)
self.log.info("==========Finished CrashTest setup========")
