def test_invalid_name_collection(self):
for _ in range(1000):
scope_name = BucketUtils.get_random_name(invalid_name=True)
try:
status, content = BucketHelper(
self.cluster.master).create_scope(self.bucket, scope_name)
if status is True:
self.log_failure("Scope '%s::%s' creation not failed: %s" %
(self.bucket, scope_name, content))
except Exception as e:
self.log.debug(e)
for _ in range(1000):
collection_name = BucketUtils.get_random_name(invalid_name=True)
try:
status, content = BucketHelper(self.cluster.master) \
.create_collection(self.bucket, CbServer.default_scope,
collection_name)
if status is True:
self.log_failure(
"Collection '%s::%s::%s' creation not failed: %s" %
(self.bucket, CbServer.default_scope, collection_name,
content))
except Exception as e:
self.log.debug(e)
# Validate doc count as per bucket collections
self.bucket_util.validate_docs_per_collections_all_buckets()
self.validate_test_failure()
def test_create_remove_collection_with_node_crash(self):
"""
1. Select a error scenario to simulate in random
2. Create error scenario either before or after collection action
3. Initiate collection creation/deletion under the bucket
4. Validate the outcome of collection creation/deletion
"""
def create_collection(client_type, bucket_obj, scope, collection):
if client_type == "sdk":
client.create_collection(collection, scope)
self.bucket_util.create_collection_object(bucket_obj, scope,
{"name": collection})
elif client_type == "rest":
self.bucket_util.create_collection(self.cluster.master,
bucket_obj,
scope,
{"name": collection})
else:
self.log_failure("Invalid client_type provided")
def remove_collection(client_type, bucket_obj, scope, collection):
if client_type == "sdk":
client.drop_collection(scope, collection)
self.bucket_util.mark_collection_as_dropped(bucket_obj, scope,
collection)
elif client_type == "rest":
self.bucket_util.drop_collection(self.cluster.master,
bucket_obj, scope, collection)
else:
self.log_failure("Invalid client_type provided")
kv_nodes = self.cluster_util.get_kv_nodes()
if len(kv_nodes) == 1:
self.fail("Need atleast two KV nodes to run this test")
client = None
task = None
action = self.input.param("action", "create")
crash_during = self.input.param("crash_during", "pre_action")
data_load_option = self.input.param("data_load_option", None)
crash_type = self.input.param("simulate_error",
CouchbaseError.KILL_MEMCACHED)
if self.scope_name != CbServer.default_scope:
self.scope_name = \
BucketUtils.get_random_name(
max_length=CbServer.max_scope_name_len)
self.bucket_util.create_scope(self.cluster.master, self.bucket,
{"name": self.scope_name})
if self.collection_name != CbServer.default_collection:
self.collection_name = \
BucketUtils.get_random_name(
max_length=CbServer.max_collection_name_len)
# Select a KV node other than master node from the cluster
node_to_crash = kv_nodes[sample(range(1, len(kv_nodes)), 1)[0]]
client = self.sdk_client_pool.get_client_for_bucket(self.bucket)
use_client = sample(["sdk", "rest"], 1)[0]
if action == "remove" \
and self.collection_name != CbServer.default_collection:
# Create a collection to be removed
create_collection(use_client, self.bucket,
self.scope_name, self.collection_name)
# Create a error scenario
self.log.info("Selected scenario for test '%s'" % crash_type)
shell = RemoteMachineShellConnection(node_to_crash)
cb_error = CouchbaseError(self.log, shell)
cbstat_obj = Cbstats(shell)
active_vbs = cbstat_obj.vbucket_list(self.bucket.name,
vbucket_type="active")
target_vbuckets = list(
set(range(0, 1024)).difference(set(active_vbs)))
doc_gen = doc_generator(self.key, 0, 1000,
target_vbucket=target_vbuckets)
if crash_during == "pre_action":
cb_error.create(crash_type)
if data_load_option == "mutate_default_collection":
task = self.task.async_load_gen_docs(
self.cluster, self.bucket, doc_gen,
DocLoading.Bucket.DocOps.UPDATE,
exp=self.maxttl,
batch_size=200, process_concurrency=8,
compression=self.sdk_compression,
durability=self.durability_level,
timeout_secs=self.sdk_timeout)
if action == "create":
create_collection(self.client_type, self.bucket,
self.scope_name, self.collection_name)
elif action == "remove":
remove_collection(self.client_type, self.bucket,
self.scope_name, self.collection_name)
if crash_during == "post_action":
cb_error.create(crash_type)
if data_load_option == "mutate_default_collection":
self.task_manager.get_task_result(task)
self.sleep(60, "Wait before reverting the error scenario")
cb_error.revert(crash_type)
# Close SSH and SDK connections
shell.disconnect()
if self.atomicity is False:
self.bucket_util.validate_docs_per_collections_all_buckets(
self.cluster)
self.validate_test_failure()
def setUp(self):
super(basic_ops, self).setUp()
# Scope/collection name can be default or create a random one to test
self.scope_name = self.input.param("scope", CbServer.default_scope)
self.collection_name = self.input.param("collection",
CbServer.default_collection)
nodes_init = self.cluster.servers[1:self.nodes_init] \
if self.nodes_init != 1 else []
self.task.rebalance([self.cluster.master], nodes_init, [])
self.cluster.nodes_in_cluster.extend([self.cluster.master] +
nodes_init)
self.bucket_util.create_default_bucket(
replica=self.num_replicas,
compression_mode=self.compression_mode,
bucket_type=self.bucket_type,
storage=self.bucket_storage,
eviction_policy=self.bucket_eviction_policy)
self.bucket_util.add_rbac_user()
# Create Scope/Collection with random names if not equal to default
if self.scope_name != CbServer.default_scope:
self.scope_name = BucketUtils.get_random_name()
BucketUtils.create_scope(self.cluster.master,
self.bucket_util.buckets[0],
{"name": self.scope_name})
if self.collection_name != CbServer.default_collection:
self.collection_name = BucketUtils.get_random_name()
BucketUtils.create_collection(self.cluster.master,
self.bucket_util.buckets[0],
self.scope_name, {
"name": self.collection_name,
"num_items": self.num_items
})
self.log.info("Using scope::collection - '%s::%s'" %
(self.scope_name, self.collection_name))
# Update required num_items under default collection
self.bucket_util.buckets[0] \
.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items = self.num_items
if self.sdk_client_pool:
self.log.info("Creating SDK client pool")
self.sdk_client_pool.create_clients(
self.bucket_util.buckets[0],
self.cluster.nodes_in_cluster,
req_clients=self.task_manager.number_of_threads,
compression_settings=self.sdk_compression)
self.durability_helper = DurabilityHelper(
self.log,
len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
# Reset active_resident_threshold to avoid further data load as DGM
self.active_resident_threshold = 0
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.log.info("==========Finished Basic_ops base setup========")
def setUp(self):
super(RebalanceBaseTest, self).setUp()
self.rest = RestConnection(self.cluster.master)
self.doc_ops = self.input.param("doc_ops", "create")
self.key_size = self.input.param("key_size", 0)
self.zone = self.input.param("zone", 1)
self.replica_to_update = self.input.param("new_replica", None)
self.default_view_name = "default_view"
self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}"
self.default_view = View(self.default_view_name, self.defaul_map_func,
None)
self.max_verify = self.input.param("max_verify", None)
self.std_vbucket_dist = self.input.param("std_vbucket_dist", None)
self.flusher_total_batch_limit = self.input.param(
"flusher_total_batch_limit", None)
self.test_abort_snapshot = self.input.param("test_abort_snapshot",
False)
self.items = self.num_items
self.logs_folder = self.input.param("logs_folder")
node_ram_ratio = self.bucket_util.base_bucket_ratio(
self.cluster.servers)
info = self.rest.get_nodes_self()
self.rest.init_cluster(username=self.cluster.master.rest_username,
password=self.cluster.master.rest_password)
self.rest.init_cluster_memoryQuota(
memoryQuota=int(info.mcdMemoryReserved * node_ram_ratio))
self.check_temporary_failure_exception = False
nodes_init = self.cluster.servers[1:self.nodes_init] \
if self.nodes_init != 1 else []
services = None
if self.services_init:
services = list()
for service in self.services_init.split("-"):
services.append(service.replace(":", ","))
services = services[1:] if len(services) > 1 else None
if nodes_init:
result = self.task.rebalance([self.cluster.master],
nodes_init, [],
services=services)
self.assertTrue(result, "Initial rebalance failed")
self.cluster.nodes_in_cluster.extend([self.cluster.master] +
nodes_init)
self.check_replica = self.input.param("check_replica", False)
self.spec_name = self.input.param("bucket_spec", None)
self.bucket_util.add_rbac_user()
# Buckets creation and initial data load done by bucket_spec
if self.spec_name is not None:
try:
self.collection_setup()
except Java_base_exception as exception:
self.handle_setup_exception(exception)
except Exception as exception:
self.handle_setup_exception(exception)
else:
if self.standard_buckets > 10:
self.bucket_util.change_max_buckets(self.standard_buckets)
self.create_buckets(self.bucket_size)
# Create Scope/Collection based on inputs given
for bucket in self.bucket_util.buckets:
if self.scope_name != CbServer.default_scope:
self.scope_name = BucketUtils.get_random_name()
BucketUtils.create_scope(self.cluster.master, bucket,
{"name": self.scope_name})
if self.collection_name != CbServer.default_collection:
self.collection_name = BucketUtils.get_random_name()
BucketUtils.create_collection(
self.cluster.master, bucket, self.scope_name, {
"name": self.collection_name,
"num_items": self.num_items
})
self.log.info(
"Bucket %s using scope::collection - '%s::%s'" %
(bucket.name, self.scope_name, self.collection_name))
# Update required num_items under default collection
bucket.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items = self.num_items
if self.flusher_total_batch_limit:
self.bucket_util.set_flusher_total_batch_limit(
self.cluster.master, self.flusher_total_batch_limit,
self.bucket_util.buckets)
self.gen_create = self.get_doc_generator(0, self.num_items)
if self.active_resident_threshold < 100:
self.check_temporary_failure_exception = True
# Reset num_items=0 since the num_items will be populated
# by the DGM load task
for bucket in self.bucket_util.buckets:
bucket.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items = 0
# Create clients in SDK client pool
if self.sdk_client_pool:
self.log.info("Creating SDK clients for client_pool")
for bucket in self.bucket_util.buckets:
self.sdk_client_pool.create_clients(
bucket, [self.cluster.master],
self.sdk_pool_capacity,
compression_settings=self.sdk_compression)
if not self.atomicity:
_ = self._load_all_buckets(self.cluster,
self.gen_create,
"create",
0,
batch_size=self.batch_size)
self.log.info("Verifying num_items counts after doc_ops")
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.validate_docs_per_collections_all_buckets(
timeout=self.wait_timeout)
else:
self.transaction_commit = True
self._load_all_buckets_atomicty(self.gen_create, "create")
self.transaction_commit = self.input.param(
"transaction_commit", True)
# Initialize doc_generators
self.active_resident_threshold = 100
self.gen_create = None
self.gen_delete = None
self.gen_update = self.get_doc_generator(0, (self.items / 2))
self.durability_helper = DurabilityHelper(
self.log,
len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.log_setup_status("RebalanceBase", "complete")
def setUp(self):
super(RebalanceBaseTest, self).setUp()
self.rest = RestConnection(self.cluster.master)
self.doc_ops = self.input.param("doc_ops", "create")
self.key_size = self.input.param("key_size", 0)
self.zone = self.input.param("zone", 1)
self.replica_to_update = self.input.param("new_replica", None)
self.default_view_name = "default_view"
self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}"
self.default_view = View(self.default_view_name, self.defaul_map_func,
None)
self.max_verify = self.input.param("max_verify", None)
self.std_vbucket_dist = self.input.param("std_vbucket_dist", None)
self.flusher_total_batch_limit = self.input.param("flusher_total_batch_limit", None)
self.test_abort_snapshot = self.input.param("test_abort_snapshot",
False)
self.items = self.num_items
node_ram_ratio = self.bucket_util.base_bucket_ratio(self.cluster.servers)
info = self.rest.get_nodes_self()
self.rest.init_cluster(username=self.cluster.master.rest_username,
password=self.cluster.master.rest_password)
self.rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved*node_ram_ratio))
self.check_temporary_failure_exception = False
nodes_init = self.cluster.servers[1:self.nodes_init] if self.nodes_init != 1 else []
if nodes_init:
result = self.task.rebalance([self.cluster.master], nodes_init, [])
self.assertTrue(result, "Initial rebalance failed")
self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init)
self.check_replica = self.input.param("check_replica", False)
self.spec_name = self.input.param("bucket_spec", None)
# If buckets creation and initial data load is to be done by bucket_spec
if self.spec_name is not None:
self.log.info("Creating buckets from spec")
# Create bucket(s) and add rbac user
buckets_spec = self.bucket_util.get_bucket_template_from_package(
self.spec_name)
doc_loading_spec = \
self.bucket_util.get_crud_template_from_package("initial_load")
self.bucket_util.create_buckets_using_json_data(buckets_spec)
self.bucket_util.wait_for_collection_creation_to_complete()
# Create clients in SDK client pool
if self.sdk_client_pool:
self.log.info("Creating required SDK clients for client_pool")
bucket_count = len(self.bucket_util.buckets)
max_clients = self.task_manager.number_of_threads
clients_per_bucket = int(ceil(max_clients / bucket_count))
for bucket in self.bucket_util.buckets:
self.sdk_client_pool.create_clients(
bucket,
[self.cluster.master],
clients_per_bucket,
compression_settings=self.sdk_compression)
self.bucket_util.run_scenario_from_spec(self.task,
self.cluster,
self.bucket_util.buckets,
doc_loading_spec,
mutation_num=0)
self.bucket_util.add_rbac_user()
self.cluster_util.print_cluster_stats()
# Verify initial doc load count
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.validate_docs_per_collections_all_buckets()
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.bucket_helper_obj = BucketHelper(self.cluster.master)
self.log.info("==========Finished rebalance base setup========")
else:
self.bucket_util.add_rbac_user()
if self.standard_buckets > 10:
self.bucket_util.change_max_buckets(self.standard_buckets)
self.create_buckets(self.bucket_size)
# Create Scope/Collection based on inputs given
for bucket in self.bucket_util.buckets:
if self.scope_name != CbServer.default_scope:
self.scope_name = BucketUtils.get_random_name()
BucketUtils.create_scope(self.cluster.master,
bucket,
{"name": self.scope_name})
if self.collection_name != CbServer.default_collection:
self.collection_name = BucketUtils.get_random_name()
BucketUtils.create_collection(self.cluster.master,
bucket,
self.scope_name,
{"name": self.collection_name,
"num_items": self.num_items})
self.log.info("Bucket %s using scope::collection - '%s::%s'"
% (bucket.name,
self.scope_name,
self.collection_name))
# Update required num_items under default collection
bucket.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items = self.num_items
if self.flusher_total_batch_limit:
self.bucket_util.set_flusher_total_batch_limit(
self.cluster.master,
self.flusher_total_batch_limit,
self.bucket_util.buckets)
self.gen_create = self.get_doc_generator(0, self.num_items)
if self.active_resident_threshold < 100:
self.check_temporary_failure_exception = True
if not self.atomicity:
_ = self._load_all_buckets(self.cluster, self.gen_create,
"create", 0, batch_size=self.batch_size)
self.log.info("Verifying num_items counts after doc_ops")
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.validate_docs_per_collections_all_buckets(
timeout=120)
else:
self.transaction_commit = True
self._load_all_buckets_atomicty(self.gen_create, "create")
self.transaction_commit = self.input.param("transaction_commit",
True)
# Initialize doc_generators
self.active_resident_threshold = 100
self.gen_create = None
self.gen_delete = None
self.gen_update = self.get_doc_generator(0, (self.items / 2))
self.durability_helper = DurabilityHelper(
self.log, len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to, persist_to=self.persist_to)
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.log.info("==========Finished rebalance base setup========")
def create_delete_collections(self):
"""
1. Create Scope-Collection
2. Validate '_default' collection values are intact
3. Load documents into created collection
4. Validate documents are loaded in new collection
5. Delete the collection and validate the '_default' collection
is unaffected
"""
use_scope_name_for_collection = \
self.input.param("use_scope_name_for_collection", False)
scope_name = BucketUtils.get_random_name()
collection_name = scope_name
if not use_scope_name_for_collection:
collection_name = BucketUtils.get_random_name()
gen_add = doc_generator(self.key, 0, self.num_items)
gen_set = doc_generator(self.key,
0,
self.num_items,
mutate=1,
mutation_type='SET')
self.log.info("Creating scope::collection '%s::%s'" %
(scope_name, collection_name))
self.bucket_util.create_scope(self.cluster.master, self.bucket,
scope_name)
self.bucket_util.create_collection(self.cluster.master, self.bucket,
scope_name, collection_name)
self.bucket_util.create_collection(self.cluster.master, self.bucket,
scope_name, "my_collection_2")
shell_conn = RemoteMachineShellConnection(self.cluster.master)
cbstats = Cbstats(shell_conn)
cbstats.get_collections(self.bucket_util.buckets[0])
self.log.info("Validating the documents in default collection")
self.bucket_util._wait_for_stats_all_buckets()
# Prints bucket stats after doc_ops
self.bucket_util.print_bucket_stats()
self.bucket_util.verify_stats_all_buckets(self.num_items)
self.log.info("Load documents into the created collection")
sdk_client = SDKClient([self.cluster.master],
self.bucket,
scope=scope_name,
collection=collection_name,
compression_settings=self.sdk_compression)
while gen_add.has_next():
key, value = gen_add.next()
result = sdk_client.crud("create",
key,
value,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout=self.sdk_timeout)
if result["status"] is False:
self.log_failure("Doc create failed for collection: %s" %
result)
break
sdk_client.close()
self.validate_test_failure()
self.bucket.scopes[scope_name] \
.collections[collection_name] \
.num_items += self.num_items
self.bucket_util._wait_for_stats_all_buckets()
# Prints bucket stats after doc_ops
self.bucket_util.print_bucket_stats()
self.bucket_util.verify_stats_all_buckets(self.num_items * 2)
task = self.task.async_load_gen_docs(self.cluster,
self.bucket,
gen_set,
"update",
0,
batch_size=10,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
compression=self.sdk_compression,
timeout_secs=self.sdk_timeout,
scope=scope_name,
collection=collection_name)
self.task_manager.get_task_result(task)
self.bucket_util._wait_for_stats_all_buckets()
# Prints bucket stats after doc_ops
self.bucket_util.print_bucket_stats()
self.bucket_util.verify_stats_all_buckets(self.num_items * 2)
self.validate_test_failure()
def test_create_remove_scope_with_node_crash(self):
"""
1. Select a error scenario to simulate in random
2. Create error scenario either before or after scope create/delete
3. Initiate scope creation/deletion under the bucket
4. Validate the outcome of scope creation/deletion
"""
def create_scope(client_type, bucket_obj, scope):
if client_type == "sdk":
client.create_scope(scope)
elif client_type == "rest":
self.bucket_util.create_scope(self.cluster.master, bucket_obj,
{"name": scope})
else:
self.log_failure("Invalid client_type provided")
def remove_scope(client_type, bucket_obj, scope):
if client_type == "sdk":
client.drop_scope(scope)
elif client_type == "rest":
self.bucket_util.drop_scope(self.cluster.master,
bucket_obj,
scope)
else:
self.log_failure("Invalid client_type provided")
kv_nodes = self.cluster_util.get_kv_nodes()
if len(kv_nodes) == 1:
self.fail("Need atleast two KV nodes to run this test")
client = None
action = self.input.param("action", "create")
crash_during = self.input.param("crash_during", "pre_action")
data_load_option = self.input.param("data_load_option", None)
crash_type = self.input.param("simulate_error",
CouchbaseError.KILL_MEMCACHED)
# Always use a random scope name to create/remove
# since CREATE/DROP not supported for default scope
self.scope_name = BucketUtils.get_random_name()
# Select a KV node other than master node from the cluster
node_to_crash = kv_nodes[sample(range(1, len(kv_nodes)), 1)[0]]
# Create a required client object
if self.client_type == "sdk":
client = SDKClient([self.cluster.master], self.bucket)
if action == "remove":
# Create a scope to be removed
use_client = sample(["sdk", "rest"], 1)[0]
create_scope(use_client, self.bucket, self.scope_name)
# Create a error scenario
shell = RemoteMachineShellConnection(node_to_crash)
cb_error = CouchbaseError(self.log, shell)
cbstat_obj = Cbstats(shell)
active_vbs = cbstat_obj.vbucket_list(self.bucket.name,
vbucket_type="active")
target_vbuckets = list(
set(range(0, 1024)).difference(set(active_vbs)))
doc_gen = doc_generator(self.key, 0, 1000,
target_vbucket=target_vbuckets)
if crash_during == "pre_action":
cb_error.create(crash_type)
if action == "create":
create_scope(self.client_type, self.bucket, self.scope_name)
elif action == "remove":
remove_scope(self.client_type, self.bucket, self.scope_name)
if crash_during == "post_action":
cb_error.create(crash_type)
if data_load_option == "mutate_default_collection":
task = self.task.async_load_gen_docs(
self.cluster, self.bucket, doc_gen, "update",
exp=self.maxttl,
batch_size=200, process_concurrency=8,
compression=self.sdk_compression,
durability=self.durability_level,
timeout_secs=self.sdk_timeout)
self.task_manager.get_task_result(task)
self.sleep(60, "Wait before reverting the error scenario")
cb_error.revert(crash_type)
# Close SSH and SDK connections
shell.disconnect()
if self.client_type == "sdk":
client.close()
self.bucket_util.validate_docs_per_collections_all_buckets()
self.validate_test_failure()
