def test_invalid_name_collection(self): for _ in range(1000): scope_name = BucketUtils.get_random_name(invalid_name=True) try: status, content = BucketHelper( self.cluster.master).create_scope(self.bucket, scope_name) if status is True: self.log_failure("Scope '%s::%s' creation not failed: %s" % (self.bucket, scope_name, content)) except Exception as e: self.log.debug(e) for _ in range(1000): collection_name = BucketUtils.get_random_name(invalid_name=True) try: status, content = BucketHelper(self.cluster.master) \ .create_collection(self.bucket, CbServer.default_scope, collection_name) if status is True: self.log_failure( "Collection '%s::%s::%s' creation not failed: %s" % (self.bucket, CbServer.default_scope, collection_name, content)) except Exception as e: self.log.debug(e) # Validate doc count as per bucket collections self.bucket_util.validate_docs_per_collections_all_buckets() self.validate_test_failure()
def test_create_remove_collection_with_node_crash(self): """ 1. Select a error scenario to simulate in random 2. Create error scenario either before or after collection action 3. Initiate collection creation/deletion under the bucket 4. Validate the outcome of collection creation/deletion """ def create_collection(client_type, bucket_obj, scope, collection): if client_type == "sdk": client.create_collection(collection, scope) self.bucket_util.create_collection_object(bucket_obj, scope, {"name": collection}) elif client_type == "rest": self.bucket_util.create_collection(self.cluster.master, bucket_obj, scope, {"name": collection}) else: self.log_failure("Invalid client_type provided") def remove_collection(client_type, bucket_obj, scope, collection): if client_type == "sdk": client.drop_collection(scope, collection) self.bucket_util.mark_collection_as_dropped(bucket_obj, scope, collection) elif client_type == "rest": self.bucket_util.drop_collection(self.cluster.master, bucket_obj, scope, collection) else: self.log_failure("Invalid client_type provided") kv_nodes = self.cluster_util.get_kv_nodes() if len(kv_nodes) == 1: self.fail("Need atleast two KV nodes to run this test") client = None task = None action = self.input.param("action", "create") crash_during = self.input.param("crash_during", "pre_action") data_load_option = self.input.param("data_load_option", None) crash_type = self.input.param("simulate_error", CouchbaseError.KILL_MEMCACHED) if self.scope_name != CbServer.default_scope: self.scope_name = \ BucketUtils.get_random_name( max_length=CbServer.max_scope_name_len) self.bucket_util.create_scope(self.cluster.master, self.bucket, {"name": self.scope_name}) if self.collection_name != CbServer.default_collection: self.collection_name = \ BucketUtils.get_random_name( max_length=CbServer.max_collection_name_len) # Select a KV node other than master node from the cluster node_to_crash = kv_nodes[sample(range(1, len(kv_nodes)), 1)[0]] client = self.sdk_client_pool.get_client_for_bucket(self.bucket) use_client = sample(["sdk", "rest"], 1)[0] if action == "remove" \ and self.collection_name != CbServer.default_collection: # Create a collection to be removed create_collection(use_client, self.bucket, self.scope_name, self.collection_name) # Create a error scenario self.log.info("Selected scenario for test '%s'" % crash_type) shell = RemoteMachineShellConnection(node_to_crash) cb_error = CouchbaseError(self.log, shell) cbstat_obj = Cbstats(shell) active_vbs = cbstat_obj.vbucket_list(self.bucket.name, vbucket_type="active") target_vbuckets = list( set(range(0, 1024)).difference(set(active_vbs))) doc_gen = doc_generator(self.key, 0, 1000, target_vbucket=target_vbuckets) if crash_during == "pre_action": cb_error.create(crash_type) if data_load_option == "mutate_default_collection": task = self.task.async_load_gen_docs( self.cluster, self.bucket, doc_gen, DocLoading.Bucket.DocOps.UPDATE, exp=self.maxttl, batch_size=200, process_concurrency=8, compression=self.sdk_compression, durability=self.durability_level, timeout_secs=self.sdk_timeout) if action == "create": create_collection(self.client_type, self.bucket, self.scope_name, self.collection_name) elif action == "remove": remove_collection(self.client_type, self.bucket, self.scope_name, self.collection_name) if crash_during == "post_action": cb_error.create(crash_type) if data_load_option == "mutate_default_collection": self.task_manager.get_task_result(task) self.sleep(60, "Wait before reverting the error scenario") cb_error.revert(crash_type) # Close SSH and SDK connections shell.disconnect() if self.atomicity is False: self.bucket_util.validate_docs_per_collections_all_buckets( self.cluster) self.validate_test_failure()
def setUp(self): super(basic_ops, self).setUp() # Scope/collection name can be default or create a random one to test self.scope_name = self.input.param("scope", CbServer.default_scope) self.collection_name = self.input.param("collection", CbServer.default_collection) nodes_init = self.cluster.servers[1:self.nodes_init] \ if self.nodes_init != 1 else [] self.task.rebalance([self.cluster.master], nodes_init, []) self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init) self.bucket_util.create_default_bucket( replica=self.num_replicas, compression_mode=self.compression_mode, bucket_type=self.bucket_type, storage=self.bucket_storage, eviction_policy=self.bucket_eviction_policy) self.bucket_util.add_rbac_user() # Create Scope/Collection with random names if not equal to default if self.scope_name != CbServer.default_scope: self.scope_name = BucketUtils.get_random_name() BucketUtils.create_scope(self.cluster.master, self.bucket_util.buckets[0], {"name": self.scope_name}) if self.collection_name != CbServer.default_collection: self.collection_name = BucketUtils.get_random_name() BucketUtils.create_collection(self.cluster.master, self.bucket_util.buckets[0], self.scope_name, { "name": self.collection_name, "num_items": self.num_items }) self.log.info("Using scope::collection - '%s::%s'" % (self.scope_name, self.collection_name)) # Update required num_items under default collection self.bucket_util.buckets[0] \ .scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items = self.num_items if self.sdk_client_pool: self.log.info("Creating SDK client pool") self.sdk_client_pool.create_clients( self.bucket_util.buckets[0], self.cluster.nodes_in_cluster, req_clients=self.task_manager.number_of_threads, compression_settings=self.sdk_compression) self.durability_helper = DurabilityHelper( self.log, len(self.cluster.nodes_in_cluster), durability=self.durability_level, replicate_to=self.replicate_to, persist_to=self.persist_to) # Reset active_resident_threshold to avoid further data load as DGM self.active_resident_threshold = 0 self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() self.log.info("==========Finished Basic_ops base setup========")
def setUp(self): super(RebalanceBaseTest, self).setUp() self.rest = RestConnection(self.cluster.master) self.doc_ops = self.input.param("doc_ops", "create") self.key_size = self.input.param("key_size", 0) self.zone = self.input.param("zone", 1) self.replica_to_update = self.input.param("new_replica", None) self.default_view_name = "default_view" self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}" self.default_view = View(self.default_view_name, self.defaul_map_func, None) self.max_verify = self.input.param("max_verify", None) self.std_vbucket_dist = self.input.param("std_vbucket_dist", None) self.flusher_total_batch_limit = self.input.param( "flusher_total_batch_limit", None) self.test_abort_snapshot = self.input.param("test_abort_snapshot", False) self.items = self.num_items self.logs_folder = self.input.param("logs_folder") node_ram_ratio = self.bucket_util.base_bucket_ratio( self.cluster.servers) info = self.rest.get_nodes_self() self.rest.init_cluster(username=self.cluster.master.rest_username, password=self.cluster.master.rest_password) self.rest.init_cluster_memoryQuota( memoryQuota=int(info.mcdMemoryReserved * node_ram_ratio)) self.check_temporary_failure_exception = False nodes_init = self.cluster.servers[1:self.nodes_init] \ if self.nodes_init != 1 else [] services = None if self.services_init: services = list() for service in self.services_init.split("-"): services.append(service.replace(":", ",")) services = services[1:] if len(services) > 1 else None if nodes_init: result = self.task.rebalance([self.cluster.master], nodes_init, [], services=services) self.assertTrue(result, "Initial rebalance failed") self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init) self.check_replica = self.input.param("check_replica", False) self.spec_name = self.input.param("bucket_spec", None) self.bucket_util.add_rbac_user() # Buckets creation and initial data load done by bucket_spec if self.spec_name is not None: try: self.collection_setup() except Java_base_exception as exception: self.handle_setup_exception(exception) except Exception as exception: self.handle_setup_exception(exception) else: if self.standard_buckets > 10: self.bucket_util.change_max_buckets(self.standard_buckets) self.create_buckets(self.bucket_size) # Create Scope/Collection based on inputs given for bucket in self.bucket_util.buckets: if self.scope_name != CbServer.default_scope: self.scope_name = BucketUtils.get_random_name() BucketUtils.create_scope(self.cluster.master, bucket, {"name": self.scope_name}) if self.collection_name != CbServer.default_collection: self.collection_name = BucketUtils.get_random_name() BucketUtils.create_collection( self.cluster.master, bucket, self.scope_name, { "name": self.collection_name, "num_items": self.num_items }) self.log.info( "Bucket %s using scope::collection - '%s::%s'" % (bucket.name, self.scope_name, self.collection_name)) # Update required num_items under default collection bucket.scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items = self.num_items if self.flusher_total_batch_limit: self.bucket_util.set_flusher_total_batch_limit( self.cluster.master, self.flusher_total_batch_limit, self.bucket_util.buckets) self.gen_create = self.get_doc_generator(0, self.num_items) if self.active_resident_threshold < 100: self.check_temporary_failure_exception = True # Reset num_items=0 since the num_items will be populated # by the DGM load task for bucket in self.bucket_util.buckets: bucket.scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items = 0 # Create clients in SDK client pool if self.sdk_client_pool: self.log.info("Creating SDK clients for client_pool") for bucket in self.bucket_util.buckets: self.sdk_client_pool.create_clients( bucket, [self.cluster.master], self.sdk_pool_capacity, compression_settings=self.sdk_compression) if not self.atomicity: _ = self._load_all_buckets(self.cluster, self.gen_create, "create", 0, batch_size=self.batch_size) self.log.info("Verifying num_items counts after doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.validate_docs_per_collections_all_buckets( timeout=self.wait_timeout) else: self.transaction_commit = True self._load_all_buckets_atomicty(self.gen_create, "create") self.transaction_commit = self.input.param( "transaction_commit", True) # Initialize doc_generators self.active_resident_threshold = 100 self.gen_create = None self.gen_delete = None self.gen_update = self.get_doc_generator(0, (self.items / 2)) self.durability_helper = DurabilityHelper( self.log, len(self.cluster.nodes_in_cluster), durability=self.durability_level, replicate_to=self.replicate_to, persist_to=self.persist_to) self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() self.log_setup_status("RebalanceBase", "complete")
def setUp(self): super(RebalanceBaseTest, self).setUp() self.rest = RestConnection(self.cluster.master) self.doc_ops = self.input.param("doc_ops", "create") self.key_size = self.input.param("key_size", 0) self.zone = self.input.param("zone", 1) self.replica_to_update = self.input.param("new_replica", None) self.default_view_name = "default_view" self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}" self.default_view = View(self.default_view_name, self.defaul_map_func, None) self.max_verify = self.input.param("max_verify", None) self.std_vbucket_dist = self.input.param("std_vbucket_dist", None) self.flusher_total_batch_limit = self.input.param("flusher_total_batch_limit", None) self.test_abort_snapshot = self.input.param("test_abort_snapshot", False) self.items = self.num_items node_ram_ratio = self.bucket_util.base_bucket_ratio(self.cluster.servers) info = self.rest.get_nodes_self() self.rest.init_cluster(username=self.cluster.master.rest_username, password=self.cluster.master.rest_password) self.rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved*node_ram_ratio)) self.check_temporary_failure_exception = False nodes_init = self.cluster.servers[1:self.nodes_init] if self.nodes_init != 1 else [] if nodes_init: result = self.task.rebalance([self.cluster.master], nodes_init, []) self.assertTrue(result, "Initial rebalance failed") self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init) self.check_replica = self.input.param("check_replica", False) self.spec_name = self.input.param("bucket_spec", None) # If buckets creation and initial data load is to be done by bucket_spec if self.spec_name is not None: self.log.info("Creating buckets from spec") # Create bucket(s) and add rbac user buckets_spec = self.bucket_util.get_bucket_template_from_package( self.spec_name) doc_loading_spec = \ self.bucket_util.get_crud_template_from_package("initial_load") self.bucket_util.create_buckets_using_json_data(buckets_spec) self.bucket_util.wait_for_collection_creation_to_complete() # Create clients in SDK client pool if self.sdk_client_pool: self.log.info("Creating required SDK clients for client_pool") bucket_count = len(self.bucket_util.buckets) max_clients = self.task_manager.number_of_threads clients_per_bucket = int(ceil(max_clients / bucket_count)) for bucket in self.bucket_util.buckets: self.sdk_client_pool.create_clients( bucket, [self.cluster.master], clients_per_bucket, compression_settings=self.sdk_compression) self.bucket_util.run_scenario_from_spec(self.task, self.cluster, self.bucket_util.buckets, doc_loading_spec, mutation_num=0) self.bucket_util.add_rbac_user() self.cluster_util.print_cluster_stats() # Verify initial doc load count self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.validate_docs_per_collections_all_buckets() self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() self.bucket_helper_obj = BucketHelper(self.cluster.master) self.log.info("==========Finished rebalance base setup========") else: self.bucket_util.add_rbac_user() if self.standard_buckets > 10: self.bucket_util.change_max_buckets(self.standard_buckets) self.create_buckets(self.bucket_size) # Create Scope/Collection based on inputs given for bucket in self.bucket_util.buckets: if self.scope_name != CbServer.default_scope: self.scope_name = BucketUtils.get_random_name() BucketUtils.create_scope(self.cluster.master, bucket, {"name": self.scope_name}) if self.collection_name != CbServer.default_collection: self.collection_name = BucketUtils.get_random_name() BucketUtils.create_collection(self.cluster.master, bucket, self.scope_name, {"name": self.collection_name, "num_items": self.num_items}) self.log.info("Bucket %s using scope::collection - '%s::%s'" % (bucket.name, self.scope_name, self.collection_name)) # Update required num_items under default collection bucket.scopes[self.scope_name] \ .collections[self.collection_name] \ .num_items = self.num_items if self.flusher_total_batch_limit: self.bucket_util.set_flusher_total_batch_limit( self.cluster.master, self.flusher_total_batch_limit, self.bucket_util.buckets) self.gen_create = self.get_doc_generator(0, self.num_items) if self.active_resident_threshold < 100: self.check_temporary_failure_exception = True if not self.atomicity: _ = self._load_all_buckets(self.cluster, self.gen_create, "create", 0, batch_size=self.batch_size) self.log.info("Verifying num_items counts after doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.validate_docs_per_collections_all_buckets( timeout=120) else: self.transaction_commit = True self._load_all_buckets_atomicty(self.gen_create, "create") self.transaction_commit = self.input.param("transaction_commit", True) # Initialize doc_generators self.active_resident_threshold = 100 self.gen_create = None self.gen_delete = None self.gen_update = self.get_doc_generator(0, (self.items / 2)) self.durability_helper = DurabilityHelper( self.log, len(self.cluster.nodes_in_cluster), durability=self.durability_level, replicate_to=self.replicate_to, persist_to=self.persist_to) self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() self.log.info("==========Finished rebalance base setup========")
def create_delete_collections(self): """ 1. Create Scope-Collection 2. Validate '_default' collection values are intact 3. Load documents into created collection 4. Validate documents are loaded in new collection 5. Delete the collection and validate the '_default' collection is unaffected """ use_scope_name_for_collection = \ self.input.param("use_scope_name_for_collection", False) scope_name = BucketUtils.get_random_name() collection_name = scope_name if not use_scope_name_for_collection: collection_name = BucketUtils.get_random_name() gen_add = doc_generator(self.key, 0, self.num_items) gen_set = doc_generator(self.key, 0, self.num_items, mutate=1, mutation_type='SET') self.log.info("Creating scope::collection '%s::%s'" % (scope_name, collection_name)) self.bucket_util.create_scope(self.cluster.master, self.bucket, scope_name) self.bucket_util.create_collection(self.cluster.master, self.bucket, scope_name, collection_name) self.bucket_util.create_collection(self.cluster.master, self.bucket, scope_name, "my_collection_2") shell_conn = RemoteMachineShellConnection(self.cluster.master) cbstats = Cbstats(shell_conn) cbstats.get_collections(self.bucket_util.buckets[0]) self.log.info("Validating the documents in default collection") self.bucket_util._wait_for_stats_all_buckets() # Prints bucket stats after doc_ops self.bucket_util.print_bucket_stats() self.bucket_util.verify_stats_all_buckets(self.num_items) self.log.info("Load documents into the created collection") sdk_client = SDKClient([self.cluster.master], self.bucket, scope=scope_name, collection=collection_name, compression_settings=self.sdk_compression) while gen_add.has_next(): key, value = gen_add.next() result = sdk_client.crud("create", key, value, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout=self.sdk_timeout) if result["status"] is False: self.log_failure("Doc create failed for collection: %s" % result) break sdk_client.close() self.validate_test_failure() self.bucket.scopes[scope_name] \ .collections[collection_name] \ .num_items += self.num_items self.bucket_util._wait_for_stats_all_buckets() # Prints bucket stats after doc_ops self.bucket_util.print_bucket_stats() self.bucket_util.verify_stats_all_buckets(self.num_items * 2) task = self.task.async_load_gen_docs(self.cluster, self.bucket, gen_set, "update", 0, batch_size=10, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, compression=self.sdk_compression, timeout_secs=self.sdk_timeout, scope=scope_name, collection=collection_name) self.task_manager.get_task_result(task) self.bucket_util._wait_for_stats_all_buckets() # Prints bucket stats after doc_ops self.bucket_util.print_bucket_stats() self.bucket_util.verify_stats_all_buckets(self.num_items * 2) self.validate_test_failure()
def test_create_remove_scope_with_node_crash(self): """ 1. Select a error scenario to simulate in random 2. Create error scenario either before or after scope create/delete 3. Initiate scope creation/deletion under the bucket 4. Validate the outcome of scope creation/deletion """ def create_scope(client_type, bucket_obj, scope): if client_type == "sdk": client.create_scope(scope) elif client_type == "rest": self.bucket_util.create_scope(self.cluster.master, bucket_obj, {"name": scope}) else: self.log_failure("Invalid client_type provided") def remove_scope(client_type, bucket_obj, scope): if client_type == "sdk": client.drop_scope(scope) elif client_type == "rest": self.bucket_util.drop_scope(self.cluster.master, bucket_obj, scope) else: self.log_failure("Invalid client_type provided") kv_nodes = self.cluster_util.get_kv_nodes() if len(kv_nodes) == 1: self.fail("Need atleast two KV nodes to run this test") client = None action = self.input.param("action", "create") crash_during = self.input.param("crash_during", "pre_action") data_load_option = self.input.param("data_load_option", None) crash_type = self.input.param("simulate_error", CouchbaseError.KILL_MEMCACHED) # Always use a random scope name to create/remove # since CREATE/DROP not supported for default scope self.scope_name = BucketUtils.get_random_name() # Select a KV node other than master node from the cluster node_to_crash = kv_nodes[sample(range(1, len(kv_nodes)), 1)[0]] # Create a required client object if self.client_type == "sdk": client = SDKClient([self.cluster.master], self.bucket) if action == "remove": # Create a scope to be removed use_client = sample(["sdk", "rest"], 1)[0] create_scope(use_client, self.bucket, self.scope_name) # Create a error scenario shell = RemoteMachineShellConnection(node_to_crash) cb_error = CouchbaseError(self.log, shell) cbstat_obj = Cbstats(shell) active_vbs = cbstat_obj.vbucket_list(self.bucket.name, vbucket_type="active") target_vbuckets = list( set(range(0, 1024)).difference(set(active_vbs))) doc_gen = doc_generator(self.key, 0, 1000, target_vbucket=target_vbuckets) if crash_during == "pre_action": cb_error.create(crash_type) if action == "create": create_scope(self.client_type, self.bucket, self.scope_name) elif action == "remove": remove_scope(self.client_type, self.bucket, self.scope_name) if crash_during == "post_action": cb_error.create(crash_type) if data_load_option == "mutate_default_collection": task = self.task.async_load_gen_docs( self.cluster, self.bucket, doc_gen, "update", exp=self.maxttl, batch_size=200, process_concurrency=8, compression=self.sdk_compression, durability=self.durability_level, timeout_secs=self.sdk_timeout) self.task_manager.get_task_result(task) self.sleep(60, "Wait before reverting the error scenario") cb_error.revert(crash_type) # Close SSH and SDK connections shell.disconnect() if self.client_type == "sdk": client.close() self.bucket_util.validate_docs_per_collections_all_buckets() self.validate_test_failure()