def test_illegal_scope_name(self): for name in self.invalid: try: BucketUtils.create_scope(self.cluster.master, self.bucket, {"name": name}) except Exception: self.log.info("Illegal scope name as expected") else: self.fail("Illegal scope name did not fail")
def test_create_scope_with_existing_name(self): BucketUtils.create_scope(self.cluster.master, self.bucket, {"name": "scope1"}) try: BucketUtils.create_scope(self.cluster.master, self.bucket, {"name": "scope1"}) except Exception as e: self.log.info("Scope creation failed as expected as there was scope1 already") else: self.fail("Scope creation did not fail even when given duplicate")
def test_illegal_collection_name(self): BucketUtils.create_scope(self.cluster.master, self.bucket, {"name": "scope1"}) for name in self.invalid: try: BucketUtils.create_collection(self.cluster.master, self.bucket, "scope1", {"name": name}) except Exception as e: self.log.info("Illegal collection name as expected") else: self.fail("Illegal collection name did not fail")
def test_more_than_max_collections_single_scope(self): BucketUtils.create_scope(self.cluster.master, self.bucket, {"name": "scope1"}) # Max_collections count, after considering default collection in setup max_collections = 1000 BucketUtils.create_collections(self.cluster, self.bucket, max_collections - 1, "scope1") try: BucketUtils.create_collections(self.cluster, self.bucket, 500, "scope1") except Exception as e: self.log.info("Creating more than max collections failed as expected") else: self.fail("Creating more than max collections did not fail")
def test_create_collection_with_existing_name(self): BucketUtils.create_scope(self.cluster.master, self.bucket, {"name": "scope1"}) BucketUtils.create_collection(self.cluster.master, self.bucket, "scope1", {"name": "collection1"}) try: BucketUtils.create_collection(self.cluster.master, self.bucket, "scope1", {"name": "collection1"}) except Exception: self.log.info("Collection creation failed as expected " "as there was collection1 already") else: self.fail("Collection creation did not fail " "even when given duplicate")
def test_more_than_max_collections_single_scope(self): BucketUtils.create_scope(self.cluster.master, self.bucket, {"name": "scope1"}) # create max collections under single scope collects_dict = BucketUtils.create_collections(self.cluster, self.bucket, self.MAX_COLLECTIONS, "scope1") actual_count = len(collects_dict) if actual_count != self.MAX_COLLECTIONS: self.fail("failed to create max number of collections") try: # create one more than the max allowed BucketUtils.create_collections(self.cluster, self.bucket, 1, "scope1") except Exception as e: self.log.info("Creating more than max collections failed as expected") else: self.fail("Creating more than max collections did not fail")
def test_create_delete_recreate_scope(self): bucket_dict = BucketUtils.get_random_scopes(self.bucket_util.buckets, "all", 1) # Delete scopes for bucket_name, scope_dict in bucket_dict.items(): bucket = BucketUtils.get_bucket_obj(self.bucket_util.buckets, bucket_name) for scope_name, _ in scope_dict["scopes"].items(): BucketUtils.drop_scope(self.cluster.master, bucket, scope_name) # Recreate scopes for bucket_name, scope_dict in bucket_dict.items(): bucket = BucketUtils.get_bucket_obj(self.bucket_util.buckets, bucket_name) for scope_name, _ in scope_dict["scopes"].items(): BucketUtils.create_scope(self.cluster.master, bucket, scope_name) # Validate doc count as per bucket collections self.bucket_util.validate_docs_per_collections_all_buckets() self.validate_test_failure()
def populate_uids(self, base_name="pre_qf"): """ Creates a scope, collection in each bucket and returns a dict like: {bucket_name:{"uid":uid, "cid":cid, "sid":sid}, ..} """ uids = dict() for bucket in self.cluster.buckets: scope_name = "custom_scope-" + base_name collection_name = "custom_collection" + base_name BucketUtils.create_scope(self.cluster.master, bucket, {"name": scope_name}) BucketUtils.create_collection(self.cluster.master, bucket, scope_name, {"name": collection_name}) uids[bucket.name] = dict() uids[bucket.name]["sid"] = BucketHelper(self.cluster.master). \ get_scope_id(bucket.name, scope_name) uids[bucket.name]["cid"] = BucketHelper(self.cluster.master). \ get_collection_id(bucket.name, scope_name, collection_name) uids[bucket.name]["uid"] = BucketHelper(self.cluster.master). \ get_bucket_manifest_uid(bucket.name) return uids
def test_create_delete_recreate_scope(self): scope_drop_fails = False bucket_dict = BucketUtils.get_random_scopes(self.bucket_util.buckets, "all", 1) # Delete scopes for bucket_name, scope_dict in bucket_dict.items(): bucket = BucketUtils.get_bucket_obj(self.bucket_util.buckets, bucket_name) for scope_name, _ in scope_dict["scopes"].items(): if scope_name == CbServer.default_scope: scope_drop_fails = True try: BucketUtils.drop_scope(self.cluster.master, bucket, scope_name) if scope_drop_fails: raise Exception("default scope deleted") except Exception as drop_exception: if scope_drop_fails \ and "delete_scope failed" in str(drop_exception): pass else: raise drop_exception # Recreate scopes for bucket_name, scope_dict in bucket_dict.items(): bucket = BucketUtils.get_bucket_obj(self.bucket_util.buckets, bucket_name) for scope_name, _ in scope_dict["scopes"].items(): # Cannot create a _default scope if scope_name == CbServer.default_collection: continue BucketUtils.create_scope(self.cluster.master, bucket, {"name": scope_name}) # Validate doc count as per bucket collections self.bucket_util.validate_docs_per_collections_all_buckets() self.validate_test_failure()
def setUp(self): super(basic_ops, self).setUp() # Scope/collection name can be default or create a random one to test self.scope_name = self.input.param("scope", CbServer.default_scope) self.collection_name = self.input.param("collection", CbServer.default_collection) nodes_init = self.cluster.servers[1:self.nodes_init] \ if self.nodes_init != 1 else [] self.task.rebalance([self.cluster.master], nodes_init, []) self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init) self.bucket_util.create_default_bucket( replica=self.num_replicas, compression_mode=self.compression_mode, bucket_type=self.bucket_type, storage=self.bucket_storage, eviction_policy=self.bucket_eviction_policy) self.bucket_util.add_rbac_user() # Create Scope/Collection with random names if not equal to default if self.scope_name != CbServer.default_scope: self.scope_name = BucketUtils.get_random_name() BucketUtils.create_scope(self.cluster.master, self.bucket_util.buckets[0], {"name": self.scope_name}) if self.collection_name != CbServer.default_collection: self.collection_name = BucketUtils.get_random_name() BucketUtils.create_collection(self.cluster.master, self.bucket_util.buckets[0], self.scope_name, { "name": self.collection_name, "num_items": self.num_items }) self.log.info("Using scope::collection - '%s::%s'" % (self.scope_name, self.collection_name)) # Update required num_items under default collection self.bucket_util.buckets[0] \ .scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items = self.num_items if self.sdk_client_pool: self.log.info("Creating SDK client pool") self.sdk_client_pool.create_clients( self.bucket_util.buckets[0], self.cluster.nodes_in_cluster, req_clients=self.task_manager.number_of_threads, compression_settings=self.sdk_compression) self.durability_helper = DurabilityHelper( self.log, len(self.cluster.nodes_in_cluster), durability=self.durability_level, replicate_to=self.replicate_to, persist_to=self.persist_to) # Reset active_resident_threshold to avoid further data load as DGM self.active_resident_threshold = 0 self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() self.log.info("==========Finished Basic_ops base setup========")
def setUp(self): super(RebalanceBaseTest, self).setUp() self.rest = RestConnection(self.cluster.master) self.doc_ops = self.input.param("doc_ops", "create") self.key_size = self.input.param("key_size", 0) self.zone = self.input.param("zone", 1) self.replica_to_update = self.input.param("new_replica", None) self.default_view_name = "default_view" self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}" self.default_view = View(self.default_view_name, self.defaul_map_func, None) self.max_verify = self.input.param("max_verify", None) self.std_vbucket_dist = self.input.param("std_vbucket_dist", None) self.flusher_total_batch_limit = self.input.param( "flusher_total_batch_limit", None) self.test_abort_snapshot = self.input.param("test_abort_snapshot", False) self.items = self.num_items self.logs_folder = self.input.param("logs_folder") node_ram_ratio = self.bucket_util.base_bucket_ratio( self.cluster.servers) info = self.rest.get_nodes_self() self.rest.init_cluster(username=self.cluster.master.rest_username, password=self.cluster.master.rest_password) self.rest.init_cluster_memoryQuota( memoryQuota=int(info.mcdMemoryReserved * node_ram_ratio)) self.check_temporary_failure_exception = False nodes_init = self.cluster.servers[1:self.nodes_init] \ if self.nodes_init != 1 else [] services = None if self.services_init: services = list() for service in self.services_init.split("-"): services.append(service.replace(":", ",")) services = services[1:] if len(services) > 1 else None if nodes_init: result = self.task.rebalance([self.cluster.master], nodes_init, [], services=services) self.assertTrue(result, "Initial rebalance failed") self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init) self.check_replica = self.input.param("check_replica", False) self.spec_name = self.input.param("bucket_spec", None) self.bucket_util.add_rbac_user() # Buckets creation and initial data load done by bucket_spec if self.spec_name is not None: try: self.collection_setup() except Java_base_exception as exception: self.handle_setup_exception(exception) except Exception as exception: self.handle_setup_exception(exception) else: if self.standard_buckets > 10: self.bucket_util.change_max_buckets(self.standard_buckets) self.create_buckets(self.bucket_size) # Create Scope/Collection based on inputs given for bucket in self.bucket_util.buckets: if self.scope_name != CbServer.default_scope: self.scope_name = BucketUtils.get_random_name() BucketUtils.create_scope(self.cluster.master, bucket, {"name": self.scope_name}) if self.collection_name != CbServer.default_collection: self.collection_name = BucketUtils.get_random_name() BucketUtils.create_collection( self.cluster.master, bucket, self.scope_name, { "name": self.collection_name, "num_items": self.num_items }) self.log.info( "Bucket %s using scope::collection - '%s::%s'" % (bucket.name, self.scope_name, self.collection_name)) # Update required num_items under default collection bucket.scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items = self.num_items if self.flusher_total_batch_limit: self.bucket_util.set_flusher_total_batch_limit( self.cluster.master, self.flusher_total_batch_limit, self.bucket_util.buckets) self.gen_create = self.get_doc_generator(0, self.num_items) if self.active_resident_threshold < 100: self.check_temporary_failure_exception = True # Reset num_items=0 since the num_items will be populated # by the DGM load task for bucket in self.bucket_util.buckets: bucket.scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items = 0 # Create clients in SDK client pool if self.sdk_client_pool: self.log.info("Creating SDK clients for client_pool") for bucket in self.bucket_util.buckets: self.sdk_client_pool.create_clients( bucket, [self.cluster.master], self.sdk_pool_capacity, compression_settings=self.sdk_compression) if not self.atomicity: _ = self._load_all_buckets(self.cluster, self.gen_create, "create", 0, batch_size=self.batch_size) self.log.info("Verifying num_items counts after doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.validate_docs_per_collections_all_buckets( timeout=self.wait_timeout) else: self.transaction_commit = True self._load_all_buckets_atomicty(self.gen_create, "create") self.transaction_commit = self.input.param( "transaction_commit", True) # Initialize doc_generators self.active_resident_threshold = 100 self.gen_create = None self.gen_delete = None self.gen_update = self.get_doc_generator(0, (self.items / 2)) self.durability_helper = DurabilityHelper( self.log, len(self.cluster.nodes_in_cluster), durability=self.durability_level, replicate_to=self.replicate_to, persist_to=self.persist_to) self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() self.log_setup_status("RebalanceBase", "complete")
def setUp(self): super(RebalanceBaseTest, self).setUp() self.rest = RestConnection(self.cluster.master) self.doc_ops = self.input.param("doc_ops", "create") self.key_size = self.input.param("key_size", 0) self.zone = self.input.param("zone", 1) self.replica_to_update = self.input.param("new_replica", None) self.default_view_name = "default_view" self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}" self.default_view = View(self.default_view_name, self.defaul_map_func, None) self.max_verify = self.input.param("max_verify", None) self.std_vbucket_dist = self.input.param("std_vbucket_dist", None) self.flusher_total_batch_limit = self.input.param("flusher_total_batch_limit", None) self.test_abort_snapshot = self.input.param("test_abort_snapshot", False) self.items = self.num_items node_ram_ratio = self.bucket_util.base_bucket_ratio(self.cluster.servers) info = self.rest.get_nodes_self() self.rest.init_cluster(username=self.cluster.master.rest_username, password=self.cluster.master.rest_password) self.rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved*node_ram_ratio)) self.check_temporary_failure_exception = False nodes_init = self.cluster.servers[1:self.nodes_init] if self.nodes_init != 1 else [] if nodes_init: result = self.task.rebalance([self.cluster.master], nodes_init, []) self.assertTrue(result, "Initial rebalance failed") self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init) self.check_replica = self.input.param("check_replica", False) self.spec_name = self.input.param("bucket_spec", None) # If buckets creation and initial data load is to be done by bucket_spec if self.spec_name is not None: self.log.info("Creating buckets from spec") # Create bucket(s) and add rbac user buckets_spec = self.bucket_util.get_bucket_template_from_package( self.spec_name) doc_loading_spec = \ self.bucket_util.get_crud_template_from_package("initial_load") self.bucket_util.create_buckets_using_json_data(buckets_spec) self.bucket_util.wait_for_collection_creation_to_complete() # Create clients in SDK client pool if self.sdk_client_pool: self.log.info("Creating required SDK clients for client_pool") bucket_count = len(self.bucket_util.buckets) max_clients = self.task_manager.number_of_threads clients_per_bucket = int(ceil(max_clients / bucket_count)) for bucket in self.bucket_util.buckets: self.sdk_client_pool.create_clients( bucket, [self.cluster.master], clients_per_bucket, compression_settings=self.sdk_compression) self.bucket_util.run_scenario_from_spec(self.task, self.cluster, self.bucket_util.buckets, doc_loading_spec, mutation_num=0) self.bucket_util.add_rbac_user() self.cluster_util.print_cluster_stats() # Verify initial doc load count self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.validate_docs_per_collections_all_buckets() self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() self.bucket_helper_obj = BucketHelper(self.cluster.master) self.log.info("==========Finished rebalance base setup========") else: self.bucket_util.add_rbac_user() if self.standard_buckets > 10: self.bucket_util.change_max_buckets(self.standard_buckets) self.create_buckets(self.bucket_size) # Create Scope/Collection based on inputs given for bucket in self.bucket_util.buckets: if self.scope_name != CbServer.default_scope: self.scope_name = BucketUtils.get_random_name() BucketUtils.create_scope(self.cluster.master, bucket, {"name": self.scope_name}) if self.collection_name != CbServer.default_collection: self.collection_name = BucketUtils.get_random_name() BucketUtils.create_collection(self.cluster.master, bucket, self.scope_name, {"name": self.collection_name, "num_items": self.num_items}) self.log.info("Bucket %s using scope::collection - '%s::%s'" % (bucket.name, self.scope_name, self.collection_name)) # Update required num_items under default collection bucket.scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items = self.num_items if self.flusher_total_batch_limit: self.bucket_util.set_flusher_total_batch_limit( self.cluster.master, self.flusher_total_batch_limit, self.bucket_util.buckets) self.gen_create = self.get_doc_generator(0, self.num_items) if self.active_resident_threshold < 100: self.check_temporary_failure_exception = True if not self.atomicity: _ = self._load_all_buckets(self.cluster, self.gen_create, "create", 0, batch_size=self.batch_size) self.log.info("Verifying num_items counts after doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.validate_docs_per_collections_all_buckets( timeout=120) else: self.transaction_commit = True self._load_all_buckets_atomicty(self.gen_create, "create") self.transaction_commit = self.input.param("transaction_commit", True) # Initialize doc_generators self.active_resident_threshold = 100 self.gen_create = None self.gen_delete = None self.gen_update = self.get_doc_generator(0, (self.items / 2)) self.durability_helper = DurabilityHelper( self.log, len(self.cluster.nodes_in_cluster), durability=self.durability_level, replicate_to=self.replicate_to, persist_to=self.persist_to) self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() self.log.info("==========Finished rebalance base setup========")
def test_create_collections(self): """ 1. Load data into '_default' collection (if required by test) 2. Create collection(s) under the specific 'scope' 3. Validate the collections are created properly 4. Validate '_default' collection is intact """ num_collections = self.input.param("num_collections", 1) use_default_scope = self.input.param("use_default_scope", False) use_scope_name_for_collection = \ self.input.param("use_scope_name_for_collection", False) scope_name = CbServer.default_scope collection_with_scope_name = use_scope_name_for_collection if use_default_scope: collection_with_scope_name = False # Create custom scope if not using 'default' scope if not use_default_scope: scope_name = self.bucket_util.get_random_name() self.log.info("Creating scope '%s'" % scope_name) BucketUtils.create_scope(self.cluster.master, self.bucket, {"name": scope_name}) if self.action_phase == "before_default_load": BucketUtils.create_collections( self.cluster, self.bucket, num_collections, scope_name, create_collection_with_scope_name=collection_with_scope_name) load_gen = doc_generator(self.key, 0, self.num_items) task = self.task.async_load_gen_docs( self.cluster, self.bucket, load_gen, "create", self.maxttl, batch_size=10, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, compression=self.sdk_compression, timeout_secs=self.sdk_timeout, scope=CbServer.default_scope, collection=CbServer.default_collection) self.bucket.scopes[CbServer.default_scope] \ .collections[CbServer.default_collection] \ .num_items += self.num_items # Create collections(s) while CRUDs are running in background if self.action_phase == "during_default_load": BucketUtils.create_collections( self.cluster, self.bucket, num_collections, scope_name, create_collection_with_scope_name=collection_with_scope_name) self.task_manager.get_task_result(task) # Doc count validation self.bucket_util._wait_for_stats_all_buckets() # Prints bucket stats after doc_ops self.bucket_util.print_bucket_stats() self.bucket_util.validate_doc_count_as_per_collections(self.bucket) self.validate_test_failure()