def test_max_key_size(self): if self.use_default_collection: self.key_size = 251 self.collection_name = CbServer.default_collection else: self.key_size = 247 self.collection_name = "collection-1" BucketUtils.create_collection( self.cluster.master, self.bucket, scope_name=CbServer.default_scope, collection_spec={"name": self.collection_name}) gen_load = doc_generator("test-max-key-size", 0, 1, key_size=self.key_size, vbuckets=self.cluster.vbuckets) task = self.task.async_load_gen_docs(self.cluster, self.bucket, gen_load, "create", self.maxttl, batch_size=20, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, collection=self.collection_name) self.task.jython_task_manager.get_task_result(task) if task.fail: self.log.info("Inserting doc key > max size failed as expected") else: self.fail("Inserting doc key greater than max key size " "succeeded when it should have failed")
def test_create_delete_recreate_collection(self): collections = BucketUtils.get_random_collections( self.bucket_util.buckets, 10, 10, 1) # delete collection for bucket_name, scope_dict in collections.iteritems(): bucket = BucketUtils.get_bucket_obj(self.bucket_util.buckets, bucket_name) scope_dict = scope_dict["scopes"] for scope_name, collection_dict in scope_dict.items(): collection_dict = collection_dict["collections"] for c_name, _ in collection_dict.items(): BucketUtils.drop_collection(self.cluster.master, bucket, scope_name, c_name) # recreate collection for bucket_name, scope_dict in collections.iteritems(): bucket = BucketUtils.get_bucket_obj(self.bucket_util.buckets, bucket_name) scope_dict = scope_dict["scopes"] for scope_name, collection_dict in scope_dict.items(): collection_dict = collection_dict["collections"] for c_name, _ in collection_dict.items(): # Cannot create a _default collection if c_name == CbServer.default_collection: continue col_obj = \ bucket.scopes[scope_name].collections[c_name] BucketUtils.create_collection(self.cluster.master, bucket, scope_name, col_obj.get_dict_object()) # Validate doc count as per bucket collections self.bucket_util.validate_docs_per_collections_all_buckets() self.validate_test_failure()
def test_illegal_collection_name(self): BucketUtils.create_scope(self.cluster.master, self.bucket, {"name": "scope1"}) for name in self.invalid: try: BucketUtils.create_collection(self.cluster.master, self.bucket, "scope1", {"name": name}) except Exception: self.log.info("Illegal collection name as expected") else: self.fail("Illegal collection name did not fail")
def test_create_collection_with_existing_name(self): BucketUtils.create_scope(self.cluster.master, self.bucket, {"name": "scope1"}) BucketUtils.create_collection(self.cluster.master, self.bucket, "scope1", {"name": "collection1"}) try: BucketUtils.create_collection(self.cluster.master, self.bucket, "scope1", {"name": "collection1"}) except Exception: self.log.info("Collection creation failed as expected " "as there was collection1 already") else: self.fail("Collection creation did not fail " "even when given duplicate")
def populate_uids(self, base_name="pre_qf"): """ Creates a scope, collection in each bucket and returns a dict like: {bucket_name:{"uid":uid, "cid":cid, "sid":sid}, ..} """ uids = dict() for bucket in self.cluster.buckets: scope_name = "custom_scope-" + base_name collection_name = "custom_collection" + base_name BucketUtils.create_scope(self.cluster.master, bucket, {"name": scope_name}) BucketUtils.create_collection(self.cluster.master, bucket, scope_name, {"name": collection_name}) uids[bucket.name] = dict() uids[bucket.name]["sid"] = BucketHelper(self.cluster.master). \ get_scope_id(bucket.name, scope_name) uids[bucket.name]["cid"] = BucketHelper(self.cluster.master). \ get_collection_id(bucket.name, scope_name, collection_name) uids[bucket.name]["uid"] = BucketHelper(self.cluster.master). \ get_bucket_manifest_uid(bucket.name) return uids
def setUp(self): super(basic_ops, self).setUp() # Scope/collection name can be default or create a random one to test self.scope_name = self.input.param("scope", CbServer.default_scope) self.collection_name = self.input.param("collection", CbServer.default_collection) nodes_init = self.cluster.servers[1:self.nodes_init] \ if self.nodes_init != 1 else [] self.task.rebalance([self.cluster.master], nodes_init, []) self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init) self.bucket_util.create_default_bucket( replica=self.num_replicas, compression_mode=self.compression_mode, bucket_type=self.bucket_type, storage=self.bucket_storage, eviction_policy=self.bucket_eviction_policy) self.bucket_util.add_rbac_user() # Create Scope/Collection with random names if not equal to default if self.scope_name != CbServer.default_scope: self.scope_name = BucketUtils.get_random_name() BucketUtils.create_scope(self.cluster.master, self.bucket_util.buckets[0], {"name": self.scope_name}) if self.collection_name != CbServer.default_collection: self.collection_name = BucketUtils.get_random_name() BucketUtils.create_collection(self.cluster.master, self.bucket_util.buckets[0], self.scope_name, { "name": self.collection_name, "num_items": self.num_items }) self.log.info("Using scope::collection - '%s::%s'" % (self.scope_name, self.collection_name)) # Update required num_items under default collection self.bucket_util.buckets[0] \ .scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items = self.num_items if self.sdk_client_pool: self.log.info("Creating SDK client pool") self.sdk_client_pool.create_clients( self.bucket_util.buckets[0], self.cluster.nodes_in_cluster, req_clients=self.task_manager.number_of_threads, compression_settings=self.sdk_compression) self.durability_helper = DurabilityHelper( self.log, len(self.cluster.nodes_in_cluster), durability=self.durability_level, replicate_to=self.replicate_to, persist_to=self.persist_to) # Reset active_resident_threshold to avoid further data load as DGM self.active_resident_threshold = 0 self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() self.log.info("==========Finished Basic_ops base setup========")
def setUp(self): super(RebalanceBaseTest, self).setUp() self.rest = RestConnection(self.cluster.master) self.doc_ops = self.input.param("doc_ops", "create") self.key_size = self.input.param("key_size", 0) self.zone = self.input.param("zone", 1) self.replica_to_update = self.input.param("new_replica", None) self.default_view_name = "default_view" self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}" self.default_view = View(self.default_view_name, self.defaul_map_func, None) self.max_verify = self.input.param("max_verify", None) self.std_vbucket_dist = self.input.param("std_vbucket_dist", None) self.flusher_total_batch_limit = self.input.param( "flusher_total_batch_limit", None) self.test_abort_snapshot = self.input.param("test_abort_snapshot", False) self.items = self.num_items self.logs_folder = self.input.param("logs_folder") node_ram_ratio = self.bucket_util.base_bucket_ratio( self.cluster.servers) info = self.rest.get_nodes_self() self.rest.init_cluster(username=self.cluster.master.rest_username, password=self.cluster.master.rest_password) self.rest.init_cluster_memoryQuota( memoryQuota=int(info.mcdMemoryReserved * node_ram_ratio)) self.check_temporary_failure_exception = False nodes_init = self.cluster.servers[1:self.nodes_init] \ if self.nodes_init != 1 else [] services = None if self.services_init: services = list() for service in self.services_init.split("-"): services.append(service.replace(":", ",")) services = services[1:] if len(services) > 1 else None if nodes_init: result = self.task.rebalance([self.cluster.master], nodes_init, [], services=services) self.assertTrue(result, "Initial rebalance failed") self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init) self.check_replica = self.input.param("check_replica", False) self.spec_name = self.input.param("bucket_spec", None) self.bucket_util.add_rbac_user() # Buckets creation and initial data load done by bucket_spec if self.spec_name is not None: try: self.collection_setup() except Java_base_exception as exception: self.handle_setup_exception(exception) except Exception as exception: self.handle_setup_exception(exception) else: if self.standard_buckets > 10: self.bucket_util.change_max_buckets(self.standard_buckets) self.create_buckets(self.bucket_size) # Create Scope/Collection based on inputs given for bucket in self.bucket_util.buckets: if self.scope_name != CbServer.default_scope: self.scope_name = BucketUtils.get_random_name() BucketUtils.create_scope(self.cluster.master, bucket, {"name": self.scope_name}) if self.collection_name != CbServer.default_collection: self.collection_name = BucketUtils.get_random_name() BucketUtils.create_collection( self.cluster.master, bucket, self.scope_name, { "name": self.collection_name, "num_items": self.num_items }) self.log.info( "Bucket %s using scope::collection - '%s::%s'" % (bucket.name, self.scope_name, self.collection_name)) # Update required num_items under default collection bucket.scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items = self.num_items if self.flusher_total_batch_limit: self.bucket_util.set_flusher_total_batch_limit( self.cluster.master, self.flusher_total_batch_limit, self.bucket_util.buckets) self.gen_create = self.get_doc_generator(0, self.num_items) if self.active_resident_threshold < 100: self.check_temporary_failure_exception = True # Reset num_items=0 since the num_items will be populated # by the DGM load task for bucket in self.bucket_util.buckets: bucket.scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items = 0 # Create clients in SDK client pool if self.sdk_client_pool: self.log.info("Creating SDK clients for client_pool") for bucket in self.bucket_util.buckets: self.sdk_client_pool.create_clients( bucket, [self.cluster.master], self.sdk_pool_capacity, compression_settings=self.sdk_compression) if not self.atomicity: _ = self._load_all_buckets(self.cluster, self.gen_create, "create", 0, batch_size=self.batch_size) self.log.info("Verifying num_items counts after doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.validate_docs_per_collections_all_buckets( timeout=self.wait_timeout) else: self.transaction_commit = True self._load_all_buckets_atomicty(self.gen_create, "create") self.transaction_commit = self.input.param( "transaction_commit", True) # Initialize doc_generators self.active_resident_threshold = 100 self.gen_create = None self.gen_delete = None self.gen_update = self.get_doc_generator(0, (self.items / 2)) self.durability_helper = DurabilityHelper( self.log, len(self.cluster.nodes_in_cluster), durability=self.durability_level, replicate_to=self.replicate_to, persist_to=self.persist_to) self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() self.log_setup_status("RebalanceBase", "complete")
def setUp(self): super(RebalanceBaseTest, self).setUp() self.rest = RestConnection(self.cluster.master) self.doc_ops = self.input.param("doc_ops", "create") self.key_size = self.input.param("key_size", 0) self.zone = self.input.param("zone", 1) self.replica_to_update = self.input.param("new_replica", None) self.default_view_name = "default_view" self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}" self.default_view = View(self.default_view_name, self.defaul_map_func, None) self.max_verify = self.input.param("max_verify", None) self.std_vbucket_dist = self.input.param("std_vbucket_dist", None) self.flusher_total_batch_limit = self.input.param("flusher_total_batch_limit", None) self.test_abort_snapshot = self.input.param("test_abort_snapshot", False) self.items = self.num_items node_ram_ratio = self.bucket_util.base_bucket_ratio(self.cluster.servers) info = self.rest.get_nodes_self() self.rest.init_cluster(username=self.cluster.master.rest_username, password=self.cluster.master.rest_password) self.rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved*node_ram_ratio)) self.check_temporary_failure_exception = False nodes_init = self.cluster.servers[1:self.nodes_init] if self.nodes_init != 1 else [] if nodes_init: result = self.task.rebalance([self.cluster.master], nodes_init, []) self.assertTrue(result, "Initial rebalance failed") self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init) self.check_replica = self.input.param("check_replica", False) self.spec_name = self.input.param("bucket_spec", None) # If buckets creation and initial data load is to be done by bucket_spec if self.spec_name is not None: self.log.info("Creating buckets from spec") # Create bucket(s) and add rbac user buckets_spec = self.bucket_util.get_bucket_template_from_package( self.spec_name) doc_loading_spec = \ self.bucket_util.get_crud_template_from_package("initial_load") self.bucket_util.create_buckets_using_json_data(buckets_spec) self.bucket_util.wait_for_collection_creation_to_complete() # Create clients in SDK client pool if self.sdk_client_pool: self.log.info("Creating required SDK clients for client_pool") bucket_count = len(self.bucket_util.buckets) max_clients = self.task_manager.number_of_threads clients_per_bucket = int(ceil(max_clients / bucket_count)) for bucket in self.bucket_util.buckets: self.sdk_client_pool.create_clients( bucket, [self.cluster.master], clients_per_bucket, compression_settings=self.sdk_compression) self.bucket_util.run_scenario_from_spec(self.task, self.cluster, self.bucket_util.buckets, doc_loading_spec, mutation_num=0) self.bucket_util.add_rbac_user() self.cluster_util.print_cluster_stats() # Verify initial doc load count self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.validate_docs_per_collections_all_buckets() self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() self.bucket_helper_obj = BucketHelper(self.cluster.master) self.log.info("==========Finished rebalance base setup========") else: self.bucket_util.add_rbac_user() if self.standard_buckets > 10: self.bucket_util.change_max_buckets(self.standard_buckets) self.create_buckets(self.bucket_size) # Create Scope/Collection based on inputs given for bucket in self.bucket_util.buckets: if self.scope_name != CbServer.default_scope: self.scope_name = BucketUtils.get_random_name() BucketUtils.create_scope(self.cluster.master, bucket, {"name": self.scope_name}) if self.collection_name != CbServer.default_collection: self.collection_name = BucketUtils.get_random_name() BucketUtils.create_collection(self.cluster.master, bucket, self.scope_name, {"name": self.collection_name, "num_items": self.num_items}) self.log.info("Bucket %s using scope::collection - '%s::%s'" % (bucket.name, self.scope_name, self.collection_name)) # Update required num_items under default collection bucket.scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items = self.num_items if self.flusher_total_batch_limit: self.bucket_util.set_flusher_total_batch_limit( self.cluster.master, self.flusher_total_batch_limit, self.bucket_util.buckets) self.gen_create = self.get_doc_generator(0, self.num_items) if self.active_resident_threshold < 100: self.check_temporary_failure_exception = True if not self.atomicity: _ = self._load_all_buckets(self.cluster, self.gen_create, "create", 0, batch_size=self.batch_size) self.log.info("Verifying num_items counts after doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.validate_docs_per_collections_all_buckets( timeout=120) else: self.transaction_commit = True self._load_all_buckets_atomicty(self.gen_create, "create") self.transaction_commit = self.input.param("transaction_commit", True) # Initialize doc_generators self.active_resident_threshold = 100 self.gen_create = None self.gen_delete = None self.gen_update = self.get_doc_generator(0, (self.items / 2)) self.durability_helper = DurabilityHelper( self.log, len(self.cluster.nodes_in_cluster), durability=self.durability_level, replicate_to=self.replicate_to, persist_to=self.persist_to) self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() self.log.info("==========Finished rebalance base setup========")