def test_illegal_scope_name(self):
for name in self.invalid:
try:
BucketUtils.create_scope(self.cluster.master, self.bucket,
{"name": name})
except Exception:
self.log.info("Illegal scope name as expected")
else:
self.fail("Illegal scope name did not fail")
def test_create_scope_with_existing_name(self):
BucketUtils.create_scope(self.cluster.master, self.bucket,
{"name": "scope1"})
try:
BucketUtils.create_scope(self.cluster.master, self.bucket,
{"name": "scope1"})
except Exception as e:
self.log.info("Scope creation failed as expected as there was scope1 already")
else:
self.fail("Scope creation did not fail even when given duplicate")
def test_illegal_collection_name(self):
BucketUtils.create_scope(self.cluster.master, self.bucket,
{"name": "scope1"})
for name in self.invalid:
try:
BucketUtils.create_collection(self.cluster.master, self.bucket,
"scope1", {"name": name})
except Exception as e:
self.log.info("Illegal collection name as expected")
else:
self.fail("Illegal collection name did not fail")
def test_more_than_max_collections_single_scope(self):
BucketUtils.create_scope(self.cluster.master, self.bucket,
{"name": "scope1"})
# Max_collections count, after considering default collection in setup
max_collections = 1000
BucketUtils.create_collections(self.cluster, self.bucket, max_collections - 1, "scope1")
try:
BucketUtils.create_collections(self.cluster, self.bucket, 500, "scope1")
except Exception as e:
self.log.info("Creating more than max collections failed as expected")
else:
self.fail("Creating more than max collections did not fail")
def test_create_collection_with_existing_name(self):
BucketUtils.create_scope(self.cluster.master, self.bucket,
{"name": "scope1"})
BucketUtils.create_collection(self.cluster.master, self.bucket,
"scope1", {"name": "collection1"})
try:
BucketUtils.create_collection(self.cluster.master, self.bucket,
"scope1", {"name": "collection1"})
except Exception:
self.log.info("Collection creation failed as expected "
"as there was collection1 already")
else:
self.fail("Collection creation did not fail "
"even when given duplicate")
def test_more_than_max_collections_single_scope(self):
BucketUtils.create_scope(self.cluster.master, self.bucket,
{"name": "scope1"})
# create max collections under single scope
collects_dict = BucketUtils.create_collections(self.cluster, self.bucket, self.MAX_COLLECTIONS, "scope1")
actual_count = len(collects_dict)
if actual_count != self.MAX_COLLECTIONS:
self.fail("failed to create max number of collections")
try:
# create one more than the max allowed
BucketUtils.create_collections(self.cluster, self.bucket, 1, "scope1")
except Exception as e:
self.log.info("Creating more than max collections failed as expected")
else:
self.fail("Creating more than max collections did not fail")
def test_create_delete_recreate_scope(self):
bucket_dict = BucketUtils.get_random_scopes(self.bucket_util.buckets,
"all", 1)
# Delete scopes
for bucket_name, scope_dict in bucket_dict.items():
bucket = BucketUtils.get_bucket_obj(self.bucket_util.buckets,
bucket_name)
for scope_name, _ in scope_dict["scopes"].items():
BucketUtils.drop_scope(self.cluster.master, bucket, scope_name)
# Recreate scopes
for bucket_name, scope_dict in bucket_dict.items():
bucket = BucketUtils.get_bucket_obj(self.bucket_util.buckets,
bucket_name)
for scope_name, _ in scope_dict["scopes"].items():
BucketUtils.create_scope(self.cluster.master, bucket,
scope_name)
# Validate doc count as per bucket collections
self.bucket_util.validate_docs_per_collections_all_buckets()
self.validate_test_failure()
def populate_uids(self, base_name="pre_qf"):
"""
Creates a scope, collection in each bucket and
returns a dict like:
{bucket_name:{"uid":uid, "cid":cid, "sid":sid}, ..}
"""
uids = dict()
for bucket in self.cluster.buckets:
scope_name = "custom_scope-" + base_name
collection_name = "custom_collection" + base_name
BucketUtils.create_scope(self.cluster.master, bucket,
{"name": scope_name})
BucketUtils.create_collection(self.cluster.master, bucket,
scope_name,
{"name": collection_name})
uids[bucket.name] = dict()
uids[bucket.name]["sid"] = BucketHelper(self.cluster.master). \
get_scope_id(bucket.name, scope_name)
uids[bucket.name]["cid"] = BucketHelper(self.cluster.master). \
get_collection_id(bucket.name, scope_name, collection_name)
uids[bucket.name]["uid"] = BucketHelper(self.cluster.master). \
get_bucket_manifest_uid(bucket.name)
return uids
def test_create_delete_recreate_scope(self):
scope_drop_fails = False
bucket_dict = BucketUtils.get_random_scopes(self.bucket_util.buckets,
"all", 1)
# Delete scopes
for bucket_name, scope_dict in bucket_dict.items():
bucket = BucketUtils.get_bucket_obj(self.bucket_util.buckets,
bucket_name)
for scope_name, _ in scope_dict["scopes"].items():
if scope_name == CbServer.default_scope:
scope_drop_fails = True
try:
BucketUtils.drop_scope(self.cluster.master, bucket,
scope_name)
if scope_drop_fails:
raise Exception("default scope deleted")
except Exception as drop_exception:
if scope_drop_fails \
and "delete_scope failed" in str(drop_exception):
pass
else:
raise drop_exception
# Recreate scopes
for bucket_name, scope_dict in bucket_dict.items():
bucket = BucketUtils.get_bucket_obj(self.bucket_util.buckets,
bucket_name)
for scope_name, _ in scope_dict["scopes"].items():
# Cannot create a _default scope
if scope_name == CbServer.default_collection:
continue
BucketUtils.create_scope(self.cluster.master, bucket,
{"name": scope_name})
# Validate doc count as per bucket collections
self.bucket_util.validate_docs_per_collections_all_buckets()
self.validate_test_failure()
def setUp(self):
super(basic_ops, self).setUp()
# Scope/collection name can be default or create a random one to test
self.scope_name = self.input.param("scope", CbServer.default_scope)
self.collection_name = self.input.param("collection",
CbServer.default_collection)
nodes_init = self.cluster.servers[1:self.nodes_init] \
if self.nodes_init != 1 else []
self.task.rebalance([self.cluster.master], nodes_init, [])
self.cluster.nodes_in_cluster.extend([self.cluster.master] +
nodes_init)
self.bucket_util.create_default_bucket(
replica=self.num_replicas,
compression_mode=self.compression_mode,
bucket_type=self.bucket_type,
storage=self.bucket_storage,
eviction_policy=self.bucket_eviction_policy)
self.bucket_util.add_rbac_user()
# Create Scope/Collection with random names if not equal to default
if self.scope_name != CbServer.default_scope:
self.scope_name = BucketUtils.get_random_name()
BucketUtils.create_scope(self.cluster.master,
self.bucket_util.buckets[0],
{"name": self.scope_name})
if self.collection_name != CbServer.default_collection:
self.collection_name = BucketUtils.get_random_name()
BucketUtils.create_collection(self.cluster.master,
self.bucket_util.buckets[0],
self.scope_name, {
"name": self.collection_name,
"num_items": self.num_items
})
self.log.info("Using scope::collection - '%s::%s'" %
(self.scope_name, self.collection_name))
# Update required num_items under default collection
self.bucket_util.buckets[0] \
.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items = self.num_items
if self.sdk_client_pool:
self.log.info("Creating SDK client pool")
self.sdk_client_pool.create_clients(
self.bucket_util.buckets[0],
self.cluster.nodes_in_cluster,
req_clients=self.task_manager.number_of_threads,
compression_settings=self.sdk_compression)
self.durability_helper = DurabilityHelper(
self.log,
len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
# Reset active_resident_threshold to avoid further data load as DGM
self.active_resident_threshold = 0
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.log.info("==========Finished Basic_ops base setup========")
def setUp(self):
super(RebalanceBaseTest, self).setUp()
self.rest = RestConnection(self.cluster.master)
self.doc_ops = self.input.param("doc_ops", "create")
self.key_size = self.input.param("key_size", 0)
self.zone = self.input.param("zone", 1)
self.replica_to_update = self.input.param("new_replica", None)
self.default_view_name = "default_view"
self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}"
self.default_view = View(self.default_view_name, self.defaul_map_func,
None)
self.max_verify = self.input.param("max_verify", None)
self.std_vbucket_dist = self.input.param("std_vbucket_dist", None)
self.flusher_total_batch_limit = self.input.param(
"flusher_total_batch_limit", None)
self.test_abort_snapshot = self.input.param("test_abort_snapshot",
False)
self.items = self.num_items
self.logs_folder = self.input.param("logs_folder")
node_ram_ratio = self.bucket_util.base_bucket_ratio(
self.cluster.servers)
info = self.rest.get_nodes_self()
self.rest.init_cluster(username=self.cluster.master.rest_username,
password=self.cluster.master.rest_password)
self.rest.init_cluster_memoryQuota(
memoryQuota=int(info.mcdMemoryReserved * node_ram_ratio))
self.check_temporary_failure_exception = False
nodes_init = self.cluster.servers[1:self.nodes_init] \
if self.nodes_init != 1 else []
services = None
if self.services_init:
services = list()
for service in self.services_init.split("-"):
services.append(service.replace(":", ","))
services = services[1:] if len(services) > 1 else None
if nodes_init:
result = self.task.rebalance([self.cluster.master],
nodes_init, [],
services=services)
self.assertTrue(result, "Initial rebalance failed")
self.cluster.nodes_in_cluster.extend([self.cluster.master] +
nodes_init)
self.check_replica = self.input.param("check_replica", False)
self.spec_name = self.input.param("bucket_spec", None)
self.bucket_util.add_rbac_user()
# Buckets creation and initial data load done by bucket_spec
if self.spec_name is not None:
try:
self.collection_setup()
except Java_base_exception as exception:
self.handle_setup_exception(exception)
except Exception as exception:
self.handle_setup_exception(exception)
else:
if self.standard_buckets > 10:
self.bucket_util.change_max_buckets(self.standard_buckets)
self.create_buckets(self.bucket_size)
# Create Scope/Collection based on inputs given
for bucket in self.bucket_util.buckets:
if self.scope_name != CbServer.default_scope:
self.scope_name = BucketUtils.get_random_name()
BucketUtils.create_scope(self.cluster.master, bucket,
{"name": self.scope_name})
if self.collection_name != CbServer.default_collection:
self.collection_name = BucketUtils.get_random_name()
BucketUtils.create_collection(
self.cluster.master, bucket, self.scope_name, {
"name": self.collection_name,
"num_items": self.num_items
})
self.log.info(
"Bucket %s using scope::collection - '%s::%s'" %
(bucket.name, self.scope_name, self.collection_name))
# Update required num_items under default collection
bucket.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items = self.num_items
if self.flusher_total_batch_limit:
self.bucket_util.set_flusher_total_batch_limit(
self.cluster.master, self.flusher_total_batch_limit,
self.bucket_util.buckets)
self.gen_create = self.get_doc_generator(0, self.num_items)
if self.active_resident_threshold < 100:
self.check_temporary_failure_exception = True
# Reset num_items=0 since the num_items will be populated
# by the DGM load task
for bucket in self.bucket_util.buckets:
bucket.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items = 0
# Create clients in SDK client pool
if self.sdk_client_pool:
self.log.info("Creating SDK clients for client_pool")
for bucket in self.bucket_util.buckets:
self.sdk_client_pool.create_clients(
bucket, [self.cluster.master],
self.sdk_pool_capacity,
compression_settings=self.sdk_compression)
if not self.atomicity:
_ = self._load_all_buckets(self.cluster,
self.gen_create,
"create",
0,
batch_size=self.batch_size)
self.log.info("Verifying num_items counts after doc_ops")
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.validate_docs_per_collections_all_buckets(
timeout=self.wait_timeout)
else:
self.transaction_commit = True
self._load_all_buckets_atomicty(self.gen_create, "create")
self.transaction_commit = self.input.param(
"transaction_commit", True)
# Initialize doc_generators
self.active_resident_threshold = 100
self.gen_create = None
self.gen_delete = None
self.gen_update = self.get_doc_generator(0, (self.items / 2))
self.durability_helper = DurabilityHelper(
self.log,
len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.log_setup_status("RebalanceBase", "complete")
def setUp(self):
super(RebalanceBaseTest, self).setUp()
self.rest = RestConnection(self.cluster.master)
self.doc_ops = self.input.param("doc_ops", "create")
self.key_size = self.input.param("key_size", 0)
self.zone = self.input.param("zone", 1)
self.replica_to_update = self.input.param("new_replica", None)
self.default_view_name = "default_view"
self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}"
self.default_view = View(self.default_view_name, self.defaul_map_func,
None)
self.max_verify = self.input.param("max_verify", None)
self.std_vbucket_dist = self.input.param("std_vbucket_dist", None)
self.flusher_total_batch_limit = self.input.param("flusher_total_batch_limit", None)
self.test_abort_snapshot = self.input.param("test_abort_snapshot",
False)
self.items = self.num_items
node_ram_ratio = self.bucket_util.base_bucket_ratio(self.cluster.servers)
info = self.rest.get_nodes_self()
self.rest.init_cluster(username=self.cluster.master.rest_username,
password=self.cluster.master.rest_password)
self.rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved*node_ram_ratio))
self.check_temporary_failure_exception = False
nodes_init = self.cluster.servers[1:self.nodes_init] if self.nodes_init != 1 else []
if nodes_init:
result = self.task.rebalance([self.cluster.master], nodes_init, [])
self.assertTrue(result, "Initial rebalance failed")
self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init)
self.check_replica = self.input.param("check_replica", False)
self.spec_name = self.input.param("bucket_spec", None)
# If buckets creation and initial data load is to be done by bucket_spec
if self.spec_name is not None:
self.log.info("Creating buckets from spec")
# Create bucket(s) and add rbac user
buckets_spec = self.bucket_util.get_bucket_template_from_package(
self.spec_name)
doc_loading_spec = \
self.bucket_util.get_crud_template_from_package("initial_load")
self.bucket_util.create_buckets_using_json_data(buckets_spec)
self.bucket_util.wait_for_collection_creation_to_complete()
# Create clients in SDK client pool
if self.sdk_client_pool:
self.log.info("Creating required SDK clients for client_pool")
bucket_count = len(self.bucket_util.buckets)
max_clients = self.task_manager.number_of_threads
clients_per_bucket = int(ceil(max_clients / bucket_count))
for bucket in self.bucket_util.buckets:
self.sdk_client_pool.create_clients(
bucket,
[self.cluster.master],
clients_per_bucket,
compression_settings=self.sdk_compression)
self.bucket_util.run_scenario_from_spec(self.task,
self.cluster,
self.bucket_util.buckets,
doc_loading_spec,
mutation_num=0)
self.bucket_util.add_rbac_user()
self.cluster_util.print_cluster_stats()
# Verify initial doc load count
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.validate_docs_per_collections_all_buckets()
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.bucket_helper_obj = BucketHelper(self.cluster.master)
self.log.info("==========Finished rebalance base setup========")
else:
self.bucket_util.add_rbac_user()
if self.standard_buckets > 10:
self.bucket_util.change_max_buckets(self.standard_buckets)
self.create_buckets(self.bucket_size)
# Create Scope/Collection based on inputs given
for bucket in self.bucket_util.buckets:
if self.scope_name != CbServer.default_scope:
self.scope_name = BucketUtils.get_random_name()
BucketUtils.create_scope(self.cluster.master,
bucket,
{"name": self.scope_name})
if self.collection_name != CbServer.default_collection:
self.collection_name = BucketUtils.get_random_name()
BucketUtils.create_collection(self.cluster.master,
bucket,
self.scope_name,
{"name": self.collection_name,
"num_items": self.num_items})
self.log.info("Bucket %s using scope::collection - '%s::%s'"
% (bucket.name,
self.scope_name,
self.collection_name))
# Update required num_items under default collection
bucket.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items = self.num_items
if self.flusher_total_batch_limit:
self.bucket_util.set_flusher_total_batch_limit(
self.cluster.master,
self.flusher_total_batch_limit,
self.bucket_util.buckets)
self.gen_create = self.get_doc_generator(0, self.num_items)
if self.active_resident_threshold < 100:
self.check_temporary_failure_exception = True
if not self.atomicity:
_ = self._load_all_buckets(self.cluster, self.gen_create,
"create", 0, batch_size=self.batch_size)
self.log.info("Verifying num_items counts after doc_ops")
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.validate_docs_per_collections_all_buckets(
timeout=120)
else:
self.transaction_commit = True
self._load_all_buckets_atomicty(self.gen_create, "create")
self.transaction_commit = self.input.param("transaction_commit",
True)
# Initialize doc_generators
self.active_resident_threshold = 100
self.gen_create = None
self.gen_delete = None
self.gen_update = self.get_doc_generator(0, (self.items / 2))
self.durability_helper = DurabilityHelper(
self.log, len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to, persist_to=self.persist_to)
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.log.info("==========Finished rebalance base setup========")
def test_create_collections(self):
"""
1. Load data into '_default' collection (if required by test)
2. Create collection(s) under the specific 'scope'
3. Validate the collections are created properly
4. Validate '_default' collection is intact
"""
num_collections = self.input.param("num_collections", 1)
use_default_scope = self.input.param("use_default_scope", False)
use_scope_name_for_collection = \
self.input.param("use_scope_name_for_collection", False)
scope_name = CbServer.default_scope
collection_with_scope_name = use_scope_name_for_collection
if use_default_scope:
collection_with_scope_name = False
# Create custom scope if not using 'default' scope
if not use_default_scope:
scope_name = self.bucket_util.get_random_name()
self.log.info("Creating scope '%s'" % scope_name)
BucketUtils.create_scope(self.cluster.master, self.bucket,
{"name": scope_name})
if self.action_phase == "before_default_load":
BucketUtils.create_collections(
self.cluster,
self.bucket,
num_collections,
scope_name,
create_collection_with_scope_name=collection_with_scope_name)
load_gen = doc_generator(self.key, 0, self.num_items)
task = self.task.async_load_gen_docs(
self.cluster,
self.bucket,
load_gen,
"create",
self.maxttl,
batch_size=10,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
compression=self.sdk_compression,
timeout_secs=self.sdk_timeout,
scope=CbServer.default_scope,
collection=CbServer.default_collection)
self.bucket.scopes[CbServer.default_scope] \
.collections[CbServer.default_collection] \
.num_items += self.num_items
# Create collections(s) while CRUDs are running in background
if self.action_phase == "during_default_load":
BucketUtils.create_collections(
self.cluster,
self.bucket,
num_collections,
scope_name,
create_collection_with_scope_name=collection_with_scope_name)
self.task_manager.get_task_result(task)
# Doc count validation
self.bucket_util._wait_for_stats_all_buckets()
# Prints bucket stats after doc_ops
self.bucket_util.print_bucket_stats()
self.bucket_util.validate_doc_count_as_per_collections(self.bucket)
self.validate_test_failure()
