def setUp(self):
super(AutoFailoverBaseTest, self).setUp()
self._get_params()
self.rest = RestConnection(self.orchestrator)
self.initial_load_gen = self.get_doc_generator(0, self.num_items)
self.update_load_gen = self.get_doc_generator(0, self.update_items)
self.delete_load_gen = self.get_doc_generator(self.update_items,
self.delete_items)
self.set_up_cluster()
self.load_all_buckets(self.initial_load_gen, "create", 0)
self.server_index_to_fail = self.input.param("server_index_to_fail",
None)
self.new_replica = self.input.param("new_replica", None)
self.replica_update_during = self.input.param("replica_update_during",
None)
if self.server_index_to_fail is None:
self.server_to_fail = self._servers_to_fail()
else:
self.server_to_fail = [
self.cluster.servers[self.server_index_to_fail]
]
self.servers_to_add = self.cluster.servers[self.
nodes_init:self.nodes_init +
self.nodes_in]
self.servers_to_remove = self.cluster.servers[self.nodes_init -
self.nodes_out:self.
nodes_init]
self.durability_helper = DurabilityHelper(self.log,
len(self.cluster.servers),
self.durability_level)
self.active_vb_in_failover_nodes = list()
self.replica_vb_in_failover_nodes = list()
self.get_vbucket_info_from_failover_nodes()
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
def setUp(self):
super(UpgradeTests, self).setUp()
self.durability_helper = DurabilityHelper(
self.log, len(self.cluster.nodes_in_cluster))
self.verification_dict = dict()
self.verification_dict["ops_create"] = self.num_items
self.verification_dict["ops_delete"] = 0
def setUp(self):
super(basic_ops, self).setUp()
self.key = 'test_docs'.rjust(self.key_size, '0')
nodes_init = self.cluster.servers[1:self.nodes_init] \
if self.nodes_init != 1 else []
self.task.rebalance([self.cluster.master], nodes_init, [])
self.cluster.nodes_in_cluster.extend([self.cluster.master] +
nodes_init)
self.bucket_util.create_default_bucket(
replica=self.num_replicas,
compression_mode=self.compression_mode,
bucket_type=self.bucket_type)
self.bucket_util.add_rbac_user()
self.src_bucket = self.bucket_util.get_all_buckets()
self.durability_helper = DurabilityHelper(
self.log,
len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
# Reset active_resident_threshold to avoid further data load as DGM
self.active_resident_threshold = 0
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.log.info("==========Finished Basic_ops base setup========")
def setUp(self):
super(CollectionBase, self).setUp()
self.log_setup_status("CollectionBase", "started")
self.key = 'test_collection'.rjust(self.key_size, '0')
self.simulate_error = self.input.param("simulate_error", None)
self.error_type = self.input.param("error_type", "memory")
self.doc_ops = self.input.param("doc_ops", None)
# If True, creates bucket/scope/collections with simpler names
self.use_simple_names = self.input.param("use_simple_names", True)
self.spec_name = self.input.param("bucket_spec",
"single_bucket.default")
self.data_spec_name = self.input.param("data_spec_name",
"initial_load")
self.remove_default_collection = \
self.input.param("remove_default_collection", False)
self.action_phase = self.input.param("action_phase",
"before_default_load")
self.skip_collections_cleanup = \
self.input.param("skip_collections_cleanup", False)
self.validate_docs_count_during_teardown = \
self.input.param("validate_docs_count_during_teardown", False)
self.batch_size = self.input.param("batch_size", 200)
self.process_concurrency = self.input.param("process_concurrency", 1)
self.retry_get_process_num = \
self.input.param("retry_get_process_num", 200)
self.change_magma_quota = self.input.param("change_magma_quota", False)
self.crud_batch_size = 100
self.num_nodes_affected = 1
if self.num_replicas > 1:
self.num_nodes_affected = 2
if self.doc_ops:
self.doc_ops = self.doc_ops.split(';')
self.durability_helper = DurabilityHelper(
self.log, len(self.cluster.nodes_in_cluster),
self.durability_level)
# Disable auto-failover to avoid failover of nodes
status = RestConnection(self.cluster.master) \
.update_autofailover_settings(False, 120)
self.assertTrue(status, msg="Failure during disabling auto-failover")
self.bucket_helper_obj = BucketHelper(self.cluster.master)
self.disk_optimized_thread_settings = self.input.param("disk_optimized_thread_settings", False)
if self.disk_optimized_thread_settings:
self.set_num_writer_and_reader_threads(num_writer_threads="disk_io_optimized",
num_reader_threads="disk_io_optimized")
try:
self.collection_setup()
except Java_base_exception as exception:
self.handle_setup_exception(exception)
except Exception as exception:
self.handle_setup_exception(exception)
self.supported_d_levels = \
self.bucket_util.get_supported_durability_levels()
self.log_setup_status("CollectionBase", "complete")
def setUp(self):
super(BucketDurabilityBase, self).setUp()
if len(self.cluster.servers) < self.nodes_init:
self.fail("Not enough nodes for rebalance")
# Disable auto-failover to avoid failover of nodes
status = RestConnection(self.cluster.master) \
.update_autofailover_settings(False, 120, False)
self.assertTrue(status, msg="Failure during disabling auto-failover")
self.durability_helper = DurabilityHelper(
self.log, len(self.cluster.nodes_in_cluster))
self.kv_nodes = self.cluster_util.get_kv_nodes(self.cluster)
self.num_nodes_affected = 1
if self.num_replicas > 1:
self.num_nodes_affected = 2
# Bucket create options representation
self.bucket_template = dict()
self.bucket_template[Bucket.name] = "default"
self.bucket_template[Bucket.ramQuotaMB] = 100
self.bucket_template[Bucket.replicaNumber] = self.num_replicas
if self.bucket_type == Bucket.Type.MEMBASE:
self.bucket_template[Bucket.storageBackend] = self.bucket_storage
# These two params will be set during each iteration
self.bucket_template[Bucket.bucketType] = None
self.bucket_template[Bucket.durabilityMinLevel] = None
self.bucket_types_to_test = [
Bucket.Type.MEMBASE, Bucket.Type.EPHEMERAL, Bucket.Type.MEMCACHED
]
self.d_level_order = [
Bucket.DurabilityLevel.NONE, Bucket.DurabilityLevel.MAJORITY,
Bucket.DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE,
Bucket.DurabilityLevel.PERSIST_TO_MAJORITY
]
# Dict representing the possible levels supported by each bucket type
self.possible_d_levels = dict()
self.possible_d_levels[Bucket.Type.MEMBASE] = \
self.bucket_util.get_supported_durability_levels()
self.possible_d_levels[Bucket.Type.EPHEMERAL] = [
Bucket.DurabilityLevel.NONE, Bucket.DurabilityLevel.MAJORITY
]
self.possible_d_levels[Bucket.Type.MEMCACHED] = [
Bucket.DurabilityLevel.NONE
]
# Dict to store the list of active/replica VBs in each node
self.vbs_in_node = dict()
for node in self.cluster_util.get_kv_nodes(self.cluster):
shell = RemoteMachineShellConnection(node)
self.vbs_in_node[node] = dict()
self.vbs_in_node[node]["shell"] = shell
self.log.info("===== BucketDurabilityBase setup complete =====")
def setUp(self):
super(RebalanceBaseTest, self).setUp()
self.doc_ops = self.input.param("doc_ops", "create")
self.doc_size = self.input.param("doc_size", 10)
self.key_size = self.input.param("key_size", 0)
self.zone = self.input.param("zone", 1)
self.new_replica = self.input.param("new_replica", None)
self.default_view_name = "default_view"
self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}"
self.default_view = View(self.default_view_name, self.defaul_map_func,
None)
self.max_verify = self.input.param("max_verify", None)
self.std_vbucket_dist = self.input.param("std_vbucket_dist", None)
self.key = 'test_docs'.rjust(self.key_size, '0')
nodes_init = self.cluster.servers[
1:self.nodes_init] if self.nodes_init != 1 else []
self.task.rebalance([self.cluster.master], nodes_init, [])
self.cluster.nodes_in_cluster.extend([self.cluster.master] +
nodes_init)
self.bucket_util.create_default_bucket(replica=self.num_replicas)
self.bucket_util.add_rbac_user()
self.sleep(10)
gen_create = self.get_doc_generator(0, self.num_items)
self.print_cluster_stat_task = self.cluster_util.async_print_cluster_stats(
)
for bucket in self.bucket_util.buckets:
task = self.task.async_load_gen_docs(
self.cluster,
bucket,
gen_create,
"create",
0,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
batch_size=10,
timeout_secs=self.sdk_timeout,
process_concurrency=8,
retries=self.sdk_retries,
durability=self.durability_level)
self.task.jython_task_manager.get_task_result(task)
self.sleep(20)
current_item = self.bucket_util.get_bucket_current_item_count(
self.cluster, bucket)
self.num_items = current_item
self.log.info("Inserted {} number of items after loadgen".format(
self.num_items))
self.gen_load = self.get_doc_generator(0, self.num_items)
# gen_update is used for doing mutation for 1/2th of uploaded data
self.gen_update = self.get_doc_generator(0, (self.num_items / 2))
self.durability_helper = DurabilityHelper(
self.log,
len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
self.log.info("==========Finished rebalance base setup========")
def setUp(self):
super(CollectionBase, self).setUp()
self.log_setup_status("CollectionBase", "started")
self.MAX_SCOPES = CbServer.max_scopes
self.MAX_COLLECTIONS = CbServer.max_collections
self.key = 'test_collection'.rjust(self.key_size, '0')
self.simulate_error = self.input.param("simulate_error", None)
self.error_type = self.input.param("error_type", "memory")
self.doc_ops = self.input.param("doc_ops", None)
self.spec_name = self.input.param("bucket_spec",
"single_bucket.default")
self.data_spec_name = self.input.param("data_spec_name",
"initial_load")
self.remove_default_collection = \
self.input.param("remove_default_collection", False)
self.action_phase = self.input.param("action_phase",
"before_default_load")
self.skip_collections_cleanup = \
self.input.param("skip_collections_cleanup", False)
self.validate_docs_count_during_teardown = \
self.input.param("validate_docs_count_during_teardown", False)
self.batch_size = self.input.param("batch_size", 200)
self.vbuckets = self.input.param("vbuckets",
self.cluster_util.vbuckets)
self.retry_get_process_num = self.input.param("retry_get_process_num",
25)
self.crud_batch_size = 100
self.num_nodes_affected = 1
if self.num_replicas > 1:
self.num_nodes_affected = 2
if self.doc_ops:
self.doc_ops = self.doc_ops.split(';')
self.durability_helper = DurabilityHelper(
self.log, len(self.cluster.nodes_in_cluster),
self.durability_level)
# Disable auto-failover to avoid failover of nodes
status = RestConnection(self.cluster.master) \
.update_autofailover_settings(False, 120, False)
self.assertTrue(status, msg="Failure during disabling auto-failover")
self.bucket_helper_obj = BucketHelper(self.cluster.master)
try:
self.collection_setup()
except Java_base_exception as exception:
self.handle_setup_exception(exception)
except Exception as exception:
self.handle_setup_exception(exception)
self.supported_d_levels = \
self.bucket_util.get_supported_durability_levels()
self.log_setup_status("CollectionBase", "complete")
def setUp(self):
super(RebalanceBaseTest, self).setUp()
self.rest = RestConnection(self.cluster.master)
self.doc_ops = self.input.param("doc_ops", "create")
self.key_size = self.input.param("key_size", 0)
self.zone = self.input.param("zone", 1)
self.replica_to_update = self.input.param("new_replica", None)
self.default_view_name = "default_view"
self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}"
self.default_view = View(self.default_view_name, self.defaul_map_func,
None)
self.max_verify = self.input.param("max_verify", None)
self.std_vbucket_dist = self.input.param("std_vbucket_dist", None)
self.flusher_total_batch_limit = self.input.param(
"flusher_total_batch_limit", None)
self.test_abort_snapshot = self.input.param("test_abort_snapshot",
False)
self.items = self.num_items
self.logs_folder = self.input.param("logs_folder")
node_ram_ratio = self.bucket_util.base_bucket_ratio(
self.cluster.servers)
info = self.rest.get_nodes_self()
self.rest.init_cluster(username=self.cluster.master.rest_username,
password=self.cluster.master.rest_password)
self.rest.init_cluster_memoryQuota(
memoryQuota=int(info.mcdMemoryReserved * node_ram_ratio))
self.check_temporary_failure_exception = False
nodes_init = self.cluster.servers[1:self.nodes_init] \
if self.nodes_init != 1 else []
services = None
if self.services_init:
services = list()
for service in self.services_init.split("-"):
services.append(service.replace(":", ","))
services = services[1:] if len(services) > 1 else None
if nodes_init:
result = self.task.rebalance([self.cluster.master],
nodes_init, [],
services=services)
self.assertTrue(result, "Initial rebalance failed")
self.cluster.nodes_in_cluster.extend([self.cluster.master] +
nodes_init)
self.check_replica = self.input.param("check_replica", False)
self.spec_name = self.input.param("bucket_spec", None)
self.bucket_util.add_rbac_user()
# Buckets creation and initial data load done by bucket_spec
if self.spec_name is not None:
try:
self.collection_setup()
except Java_base_exception as exception:
self.handle_setup_exception(exception)
except Exception as exception:
self.handle_setup_exception(exception)
else:
if self.standard_buckets > 10:
self.bucket_util.change_max_buckets(self.standard_buckets)
self.create_buckets(self.bucket_size)
# Create Scope/Collection based on inputs given
for bucket in self.bucket_util.buckets:
if self.scope_name != CbServer.default_scope:
self.scope_name = BucketUtils.get_random_name()
BucketUtils.create_scope(self.cluster.master, bucket,
{"name": self.scope_name})
if self.collection_name != CbServer.default_collection:
self.collection_name = BucketUtils.get_random_name()
BucketUtils.create_collection(
self.cluster.master, bucket, self.scope_name, {
"name": self.collection_name,
"num_items": self.num_items
})
self.log.info(
"Bucket %s using scope::collection - '%s::%s'" %
(bucket.name, self.scope_name, self.collection_name))
# Update required num_items under default collection
bucket.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items = self.num_items
if self.flusher_total_batch_limit:
self.bucket_util.set_flusher_total_batch_limit(
self.cluster.master, self.flusher_total_batch_limit,
self.bucket_util.buckets)
self.gen_create = self.get_doc_generator(0, self.num_items)
if self.active_resident_threshold < 100:
self.check_temporary_failure_exception = True
# Reset num_items=0 since the num_items will be populated
# by the DGM load task
for bucket in self.bucket_util.buckets:
bucket.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items = 0
# Create clients in SDK client pool
if self.sdk_client_pool:
self.log.info("Creating SDK clients for client_pool")
for bucket in self.bucket_util.buckets:
self.sdk_client_pool.create_clients(
bucket, [self.cluster.master],
self.sdk_pool_capacity,
compression_settings=self.sdk_compression)
if not self.atomicity:
_ = self._load_all_buckets(self.cluster,
self.gen_create,
"create",
0,
batch_size=self.batch_size)
self.log.info("Verifying num_items counts after doc_ops")
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.validate_docs_per_collections_all_buckets(
timeout=self.wait_timeout)
else:
self.transaction_commit = True
self._load_all_buckets_atomicty(self.gen_create, "create")
self.transaction_commit = self.input.param(
"transaction_commit", True)
# Initialize doc_generators
self.active_resident_threshold = 100
self.gen_create = None
self.gen_delete = None
self.gen_update = self.get_doc_generator(0, (self.items / 2))
self.durability_helper = DurabilityHelper(
self.log,
len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.log_setup_status("RebalanceBase", "complete")
def setUp(self):
super(CrashTest, self).setUp()
self.doc_ops = self.input.param("doc_ops", None)
self.process_name = self.input.param("process", None)
self.service_name = self.input.param("service", "data")
self.sig_type = self.input.param("sig_type", "SIGKILL").upper()
self.target_node = self.input.param("target_node", "active")
self.client_type = self.input.param("client_type", "sdk").lower()
self.N1qltxn = self.input.param("N1qltxn", False)
self.pre_warmup_stats = dict()
self.timeout = 120
self.new_docs_to_add = 10000
if self.doc_ops is not None:
self.doc_ops = self.doc_ops.split(";")
if not self.atomicity:
self.durability_helper = DurabilityHelper(
self.log, self.nodes_init,
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
verification_dict = dict()
verification_dict["ops_create"] = \
self.cluster.buckets[0].scopes[
CbServer.default_scope].collections[
CbServer.default_collection].num_items
verification_dict["sync_write_aborted_count"] = 0
verification_dict["rollback_item_count"] = 0
verification_dict["pending_writes"] = 0
if self.durability_level:
verification_dict["sync_write_committed_count"] = \
verification_dict["ops_create"]
# Load initial documents into the buckets
transaction_gen_create = doc_generator(
"transaction_key", 0, self.num_items,
key_size=self.key_size,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets)
gen_create = doc_generator(
self.key, 0, self.num_items,
key_size=self.key_size,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets)
if self.atomicity:
transaction_task = self.task.async_load_gen_docs_atomicity(
self.cluster, self.cluster.buckets,
transaction_gen_create, DocLoading.Bucket.DocOps.CREATE,
exp=0,
batch_size=10,
process_concurrency=self.process_concurrency,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
update_count=self.update_count,
transaction_timeout=self.transaction_timeout,
commit=True,
sync=self.sync)
self.task.jython_task_manager.get_task_result(transaction_task)
for bucket in self.cluster.buckets:
task = self.task.async_load_gen_docs(
self.cluster, bucket, gen_create,
DocLoading.Bucket.DocOps.CREATE, self.maxttl,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
batch_size=10, process_concurrency=8)
self.task.jython_task_manager.get_task_result(task)
self.bucket_util._wait_for_stats_all_buckets(self.cluster.buckets)
self.cluster.buckets[0].scopes[
CbServer.default_scope].collections[
CbServer.default_collection].num_items += self.num_items
verification_dict["ops_create"] += self.num_items
if self.durability_level:
verification_dict["sync_write_committed_count"] += \
self.num_items
# Verify cbstats vbucket-details
stats_failed = self.durability_helper.verify_vbucket_details_stats(
bucket, self.cluster_util.get_kv_nodes(),
vbuckets=self.cluster_util.vbuckets,
expected_val=verification_dict)
if self.atomicity is False:
if stats_failed:
self.fail("Cbstats verification failed")
self.bucket_util.verify_stats_all_buckets(
self.cluster,
self.cluster.buckets[0].scopes[
CbServer.default_scope].collections[
CbServer.default_collection].num_items)
self.bucket = self.cluster.buckets[0]
if self.N1qltxn:
self.n1ql_server = self.cluster_util.get_nodes_from_services_map(
service_type="n1ql",
get_all_nodes=True)
self.n1ql_helper = N1QLHelper(server=self.n1ql_server,
use_rest=True,
buckets=self.cluster.buckets,
log=self.log,
scan_consistency='REQUEST_PLUS',
num_collection=3,
num_buckets=1,
num_savepoints=1,
override_savepoint=False,
num_stmt=10,
load_spec=self.data_spec_name)
self.bucket_col = self.n1ql_helper.get_collections()
self.stmts = self.n1ql_helper.get_stmt(self.bucket_col)
self.stmts = self.n1ql_helper.create_full_stmts(self.stmts)
self.log.info("==========Finished CrashTest setup========")
def setUp(self):
super(CrashTest, self).setUp()
self.doc_ops = self.input.param("doc_ops", None)
self.process_name = self.input.param("process", None)
self.service_name = self.input.param("service", "data")
self.sig_type = self.input.param("sig_type", "SIGKILL").upper()
self.target_node = self.input.param("target_node", "active")
self.pre_warmup_stats = {}
self.timeout = 120
self.new_docs_to_add = 10000
if self.doc_ops is not None:
self.doc_ops = self.doc_ops.split(";")
nodes_init = self.cluster.servers[1:self.nodes_init] \
if self.nodes_init != 1 else []
self.task.rebalance([self.cluster.master], nodes_init, [])
self.cluster.nodes_in_cluster.extend([self.cluster.master] +
nodes_init)
if not self.atomicity:
self.durability_helper = DurabilityHelper(
self.log,
self.nodes_init,
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
self.bucket_util.create_default_bucket(
bucket_type=self.bucket_type,
ram_quota=self.bucket_size,
replica=self.num_replicas,
compression_mode="off",
storage=self.bucket_storage,
eviction_policy=self.bucket_eviction_policy)
self.bucket_util.add_rbac_user()
if self.sdk_client_pool:
self.log.info("Creating SDK clients for client_pool")
for bucket in self.bucket_util.buckets:
self.sdk_client_pool.create_clients(
bucket, [self.cluster.master],
self.sdk_pool_capacity,
compression_settings=self.sdk_compression)
verification_dict = dict()
verification_dict["ops_create"] = self.num_items
verification_dict["sync_write_aborted_count"] = 0
verification_dict["rollback_item_count"] = 0
verification_dict["pending_writes"] = 0
if self.durability_level:
verification_dict["sync_write_committed_count"] = self.num_items
# Load initial documents into the buckets
self.log.info("Loading initial documents")
gen_create = doc_generator(self.key,
0,
self.num_items,
key_size=self.key_size,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets)
if self.atomicity:
task = self.task.async_load_gen_docs_atomicity(
self.cluster,
self.bucket_util.buckets,
gen_create,
"create",
exp=0,
batch_size=10,
process_concurrency=self.process_concurrency,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
update_count=self.update_count,
transaction_timeout=self.transaction_timeout,
commit=True,
sync=self.sync)
self.task.jython_task_manager.get_task_result(task)
else:
for bucket in self.bucket_util.buckets:
task = self.task.async_load_gen_docs(
self.cluster,
bucket,
gen_create,
DocLoading.Bucket.DocOps.CREATE,
self.maxttl,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
batch_size=10,
process_concurrency=8,
sdk_client_pool=self.sdk_client_pool)
self.task.jython_task_manager.get_task_result(task)
self.bucket_util._wait_for_stats_all_buckets()
# Verify cbstats vbucket-details
stats_failed = \
self.durability_helper.verify_vbucket_details_stats(
bucket, self.cluster_util.get_kv_nodes(),
vbuckets=self.cluster_util.vbuckets,
expected_val=verification_dict)
if stats_failed:
self.fail("Cbstats verification failed")
self.bucket_util.verify_stats_all_buckets(self.num_items)
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.log.info("==========Finished CrashTest setup========")
def setUp(self):
super(DurabilityTestsBase, self).setUp()
self.simulate_error = self.input.param("simulate_error", None)
self.error_type = self.input.param("error_type", "memory")
self.doc_ops = self.input.param("doc_ops", None)
self.with_non_sync_writes = self.input.param("with_non_sync_writes",
False)
self.skip_init_load = self.input.param("skip_init_load", False)
self.crud_batch_size = 100
self.num_nodes_affected = 1
if self.num_replicas > 1:
self.num_nodes_affected = 2
if self.doc_ops:
self.doc_ops = self.doc_ops.split(';')
self.durability_helper = DurabilityHelper(
self.log, len(self.cluster.nodes_in_cluster),
self.durability_level)
# Initialize cluster using given nodes
nodes_init = self.cluster.servers[1:self.nodes_init] \
if self.nodes_init != 1 else []
self.task.rebalance([self.cluster.master], nodes_init, [])
self.cluster.nodes_in_cluster.extend([self.cluster.master]+nodes_init)
# Disable auto-failover to avoid failover of nodes
status = RestConnection(self.cluster.master) \
.update_autofailover_settings(False, 120, False)
self.assertTrue(status, msg="Failure during disabling auto-failover")
# Create default bucket and add rbac user
self.bucket_util.create_default_bucket(
replica=self.num_replicas, compression_mode=self.compression_mode,
bucket_type=self.bucket_type, storage=self.bucket_storage,
eviction_policy=self.bucket_eviction_policy)
self.bucket_util.add_rbac_user()
self.cluster_util.print_cluster_stats()
self.bucket = self.bucket_util.buckets[0]
# Create sdk_clients for pool
if self.sdk_client_pool:
self.log.info("Creating SDK client pool")
self.sdk_client_pool.create_clients(
self.bucket,
self.cluster.nodes_in_cluster,
req_clients=self.sdk_pool_capacity,
compression_settings=self.sdk_compression)
if not self.skip_init_load:
if self.target_vbucket and type(self.target_vbucket) is not list:
self.target_vbucket = [self.target_vbucket]
self.log.info("Creating doc_generator..")
doc_create = doc_generator(
self.key,
0,
self.num_items,
key_size=self.key_size,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets)
self.log.info("Loading {0} items into bucket"
.format(self.num_items))
task = self.task.async_load_gen_docs(
self.cluster, self.bucket, doc_create, "create", 0,
batch_size=10, process_concurrency=8,
replicate_to=self.replicate_to, persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
sdk_client_pool=self.sdk_client_pool)
self.task.jython_task_manager.get_task_result(task)
# Verify initial doc load count
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
self.bucket_util.print_bucket_stats()
self.log.info("=== DurabilityBaseTests setup complete ===")
def test_collection_not_exists(self):
"""
1. Load docs into required collection
2. Validate docs based on the targeted collection
3. Create non-default scope/collection for CRUDs to happen
4. Perform doc_ops again and perform CRUDs
5. Drop the target collection and validate the CollectionNotExists
exception from client side
6. Recreate non-default collection and re-create the docs and validate
"""
def validate_vb_detail_stats():
failed = durability_helper.verify_vbucket_details_stats(
self.bucket,
self.cluster_util.get_kv_nodes(),
vbuckets=self.cluster_util.vbuckets,
expected_val=verification_dict)
if failed:
self.log_failure("vBucket_details validation failed")
self.bucket_util.validate_docs_per_collections_all_buckets()
num_cols_in_bucket = 0
for _, scope in self.bucket.scopes.items():
for _, _ in scope.collections.items():
num_cols_in_bucket += 1
verification_dict = dict()
verification_dict["ops_create"] = num_cols_in_bucket * self.num_items
verification_dict["ops_update"] = 0
verification_dict["ops_delete"] = 0
verification_dict["rollback_item_count"] = 0
verification_dict["sync_write_aborted_count"] = 0
verification_dict["sync_write_committed_count"] = 0
durability_helper = DurabilityHelper(self.log,
len(self.cluster.kv_nodes),
durability=self.durability_level)
drop_scope = self.input.param("drop_scope", False)
if self.scope_name != CbServer.default_scope:
self.scope_name = self.bucket_util.get_random_name()
if self.collection_name != CbServer.default_collection:
self.collection_name = self.bucket_util.get_random_name()
# Doc generator used for mutations
doc_gen = doc_generator("test_col_not_exists", 0, 10)
# Acquire SDK client for mutations
client = self.sdk_client_pool.get_client_for_bucket(
self.bucket, self.scope_name, self.collection_name)
doc_ttl, _ = \
SDKExceptionTests.__get_random_doc_ttl_and_durability_level()
self.log.info(
"Creating docs with doc_ttl %s into %s:%s:%s" %
(doc_ttl, self.bucket.name, self.scope_name, self.collection_name))
while doc_gen.has_next():
key, value = doc_gen.next()
result = client.crud("create",
key,
value,
exp=doc_ttl,
durability=self.durability_level,
timeout=30)
if self.collection_name == CbServer.default_collection:
if result["status"] is False:
self.log_failure("Create doc failed for key: %s" % key)
else:
verification_dict["ops_create"] += 1
if self.durability_level:
verification_dict["sync_write_committed_count"] += 1
self.bucket.scopes[self.scope_name].collections[
self.collection_name].num_items += 1
elif result["status"] is True:
self.log_failure("Create didn't fail as expected for key: %s" %
key)
elif SDKException.AmbiguousTimeoutException \
not in str(result["error"]) \
or SDKException.RetryReason.COLLECTION_NOT_FOUND \
not in str(result["error"]):
self.log_failure("Invalid exception for key %s: %s" %
(key, result["error"]))
validate_vb_detail_stats()
# Create required scope/collection for successful CRUD operation
self.create_scope_collection()
# Reset doc_gen itr value for retry purpose
doc_gen.reset()
doc_ttl, _ = \
SDKExceptionTests.__get_random_doc_ttl_and_durability_level()
self.log.info(
"Creating docs with doc_ttl %s into %s:%s:%s" %
(doc_ttl, self.bucket.name, self.scope_name, self.collection_name))
op_type = "create"
if self.collection_name == CbServer.default_collection:
op_type = "update"
while doc_gen.has_next():
key, value = doc_gen.next()
result = client.crud(op_type,
key,
value,
exp=doc_ttl,
durability=self.durability_level)
if result["status"] is False:
self.log_failure("Create fail for key %s: %s" % (key, result))
else:
if op_type == "create":
verification_dict["ops_create"] += 1
self.bucket.scopes[self.scope_name].collections[
self.collection_name].num_items += 1
else:
verification_dict["ops_update"] += 1
if self.durability_level:
verification_dict["sync_write_committed_count"] += 1
validate_vb_detail_stats()
self.validate_test_failure()
if drop_scope:
self.log.info("Dropping scope %s" % self.scope_name)
self.bucket_util.drop_scope(self.cluster.master, self.bucket,
self.scope_name)
else:
self.log.info("Dropping collection %s:%s" %
(self.scope_name, self.collection_name))
self.bucket_util.drop_collection(self.cluster.master, self.bucket,
self.scope_name,
self.collection_name)
validate_vb_detail_stats()
self.validate_test_failure()
# Reset doc_gen itr value for retry purpose
doc_gen.reset()
while doc_gen.has_next():
key, value = doc_gen.next()
result = client.crud("create",
key,
value,
exp=doc_ttl,
durability=self.durability_level)
if result["status"] is True:
self.log_failure("Create doc succeeded for dropped collection")
validate_vb_detail_stats()
self.validate_test_failure()
# Re-create the dropped collection
self.create_scope_collection(create_scope=drop_scope)
if self.collection_name != CbServer.default_collection:
doc_gen.reset()
while doc_gen.has_next():
key, value = doc_gen.next()
result = client.crud("create",
key,
value,
exp=doc_ttl,
durability=self.durability_level)
if result["status"] is False:
self.log_failure("Create failed after collection recreate "
"for key %s: %s" % (key, result["error"]))
else:
verification_dict["ops_create"] += 1
if self.durability_level:
verification_dict["sync_write_committed_count"] += 1
self.bucket.scopes[self.scope_name].collections[
self.collection_name].num_items += 1
validate_vb_detail_stats()
# Release the acquired client
self.sdk_client_pool.release_client(client)
self.validate_test_failure()
def setUp(self):
super(basic_ops, self).setUp()
self.doc_ops = self.input.param("doc_ops", "").split(";")
self.observe_test = self.input.param("observe_test", False)
# Scope/collection name can be default or create a random one to test
self.scope_name = self.input.param("scope", CbServer.default_scope)
self.collection_name = self.input.param("collection",
CbServer.default_collection)
nodes_init = self.cluster.servers[1:self.nodes_init] \
if self.nodes_init != 1 else []
self.task.rebalance([self.cluster.master], nodes_init, [])
self.cluster.nodes_in_cluster.extend([self.cluster.master] +
nodes_init)
self.bucket_util.create_default_bucket(
replica=self.num_replicas,
compression_mode=self.compression_mode,
bucket_type=self.bucket_type,
storage=self.bucket_storage,
eviction_policy=self.bucket_eviction_policy)
self.bucket_util.add_rbac_user()
# Create Scope/Collection with random names if not equal to default
if self.scope_name != CbServer.default_scope:
self.scope_name = self.bucket_util.get_random_name()
self.bucket_util.create_scope(self.cluster.master,
self.bucket_util.buckets[0],
{"name": self.scope_name})
if self.collection_name != CbServer.default_collection:
self.collection_name = self.bucket_util.get_random_name()
self.bucket_util.create_collection(
self.cluster.master, self.bucket_util.buckets[0],
self.scope_name, {
"name": self.collection_name,
"num_items": self.num_items
})
self.log.info("Using scope::collection - '%s::%s'" %
(self.scope_name, self.collection_name))
# Update required num_items under default collection
self.bucket_util.buckets[0] \
.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items = self.num_items
self.durability_helper = DurabilityHelper(
self.log,
len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
# Create sdk_clients for pool
if self.sdk_client_pool:
self.log.info("Creating SDK client pool")
self.sdk_client_pool.create_clients(
self.bucket_util.buckets[0],
self.cluster.nodes_in_cluster,
req_clients=self.sdk_pool_capacity,
compression_settings=self.sdk_compression)
# Reset active_resident_threshold to avoid further data load as DGM
self.active_resident_threshold = 0
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.log.info("==========Finished Basic_ops base setup========")
def test_index_with_aborts(self):
"""
1. Create index (2i/view) on default bucket
2. Load multiple docs such that all sync_writes will be aborted
3. Verify nothing went into indexing
4. Load sync_write docs such that they are successful
5. Validate the mutated docs are taken into indexing
:return:
"""
crud_batch_size = 50
def_bucket = self.cluster.buckets[0]
kv_nodes = self.cluster_util.get_kv_nodes(self.cluster)
replica_vbs = dict()
verification_dict = dict()
index_item_count = dict()
expected_num_indexed = dict()
load_gen = dict()
load_gen["ADD"] = dict()
load_gen["SET"] = dict()
partial_aborts = ["initial_aborts", "aborts_at_end"]
durability_helper = DurabilityHelper(
self.log,
len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
if self.create_index_during == "before_doc_ops":
self.create_gsi_indexes(def_bucket)
curr_items = self.bucket_util.get_bucket_current_item_count(
self.cluster, def_bucket)
if self.sync_write_abort_pattern in ["all_aborts", "initial_aborts"]:
self.bucket_util.flush_bucket(self.cluster, def_bucket)
self.num_items = 0
else:
self.num_items = curr_items
self.log.info("Disabling auto_failover to avoid node failures")
status = RestConnection(self.cluster.master) \
.update_autofailover_settings(False, 120)
self.assertTrue(status, msg="Failure during disabling auto-failover")
# Validate vbucket stats
verification_dict["ops_create"] = self.num_items
verification_dict["ops_update"] = 0
# verification_dict["ops_delete"] = 0
verification_dict["rollback_item_count"] = 0
verification_dict["sync_write_aborted_count"] = 0
verification_dict["sync_write_committed_count"] = 0
index_item_count["#primary"] = self.num_items
index_item_count["durable_add_aborts"] = 0
index_item_count["durable_set_aborts"] = 0
expected_num_indexed["#primary"] = curr_items
expected_num_indexed["durable_add_aborts"] = 0
expected_num_indexed["durable_set_aborts"] = 0
if self.create_index_during == "before_doc_ops":
self.validate_indexed_doc_count(def_bucket, index_item_count)
self.log.info("Loading docs such that all sync_writes will be aborted")
for server in kv_nodes:
ssh_shell = RemoteMachineShellConnection(server)
cbstats = Cbstats(server)
replica_vbs[server] = cbstats.vbucket_list(def_bucket.name,
"replica")
load_gen["ADD"][server] = list()
load_gen["ADD"][server].append(
doc_generator(self.key,
0,
crud_batch_size,
target_vbucket=replica_vbs[server],
mutation_type="ADD"))
if self.sync_write_abort_pattern in partial_aborts:
load_gen["ADD"][server].append(
doc_generator(self.key,
10000,
crud_batch_size,
target_vbucket=replica_vbs[server],
mutation_type="ADD"))
verification_dict["ops_create"] += crud_batch_size
verification_dict["sync_write_committed_count"] += \
crud_batch_size
index_item_count["#primary"] += crud_batch_size
index_item_count["durable_add_aborts"] += crud_batch_size
expected_num_indexed["#primary"] += crud_batch_size
expected_num_indexed["durable_add_aborts"] += crud_batch_size
task_success = self.bucket_util.load_durable_aborts(
ssh_shell, load_gen["ADD"][server], self.cluster, def_bucket,
self.durability_level, DocLoading.Bucket.DocOps.CREATE,
self.sync_write_abort_pattern)
if not task_success:
self.log_failure("Failure during load_abort task")
verification_dict["sync_write_aborted_count"] += \
crud_batch_size
if self.create_index_during == "before_doc_ops":
self.validate_indexed_doc_count(def_bucket, index_item_count)
load_gen["SET"][server] = list()
load_gen["SET"][server].append(
doc_generator(self.key,
0,
crud_batch_size,
target_vbucket=replica_vbs[server],
mutation_type="SET"))
if self.sync_write_abort_pattern in partial_aborts:
load_gen["SET"][server].append(
doc_generator(self.key,
10000,
crud_batch_size,
target_vbucket=replica_vbs[server],
mutation_type="SET"))
verification_dict["ops_update"] += crud_batch_size
verification_dict["sync_write_committed_count"] += \
crud_batch_size
index_item_count["durable_add_aborts"] -= crud_batch_size
index_item_count["durable_set_aborts"] += crud_batch_size
expected_num_indexed["#primary"] += crud_batch_size
expected_num_indexed["durable_add_aborts"] += crud_batch_size
expected_num_indexed["durable_set_aborts"] += crud_batch_size
verification_dict["sync_write_aborted_count"] += \
crud_batch_size
task_success = self.bucket_util.load_durable_aborts(
ssh_shell, load_gen["SET"][server], self.cluster, def_bucket,
self.durability_level, DocLoading.Bucket.DocOps.UPDATE,
self.sync_write_abort_pattern)
if not task_success:
self.log_failure("Failure during load_abort task")
ssh_shell.disconnect()
if self.create_index_during == "before_doc_ops":
self.validate_indexed_doc_count(def_bucket, index_item_count)
failed = durability_helper.verify_vbucket_details_stats(
def_bucket,
kv_nodes,
vbuckets=self.cluster.vbuckets,
expected_val=verification_dict)
if failed:
self.log_failure("Cbstat vbucket-details verification failed")
self.validate_test_failure()
if self.create_index_during == "after_doc_ops":
self.create_gsi_indexes(def_bucket)
self.validate_indexed_doc_count(def_bucket, index_item_count)
self.log.info("Verify aborts are not indexed")
self.validate_indexed_count_from_stats(def_bucket,
expected_num_indexed,
index_item_count)
if not self.use_gsi_for_primary:
self.log.info("Wait of any indexing_activity to complete")
index_monitor_task = self.cluster_util.async_monitor_active_task(
self.cluster.master,
"indexer",
"_design/ddl_#primary",
num_iteration=20,
wait_task=True)[0]
self.task_manager.get_task_result(index_monitor_task)
self.assertTrue(index_monitor_task.result,
"Indexer task still running on server")
for server in kv_nodes:
if self.sync_write_abort_pattern == "initial_aborts":
load_gen["ADD"][server] = load_gen["ADD"][server][:1]
load_gen["SET"][server] = load_gen["SET"][server][:1]
elif self.sync_write_abort_pattern == "aborts_at_end":
load_gen["ADD"][server] = load_gen["ADD"][server][-1:]
load_gen["SET"][server] = load_gen["SET"][server][-1:]
self.log.info("Load sync_write docs such that they are successful")
for server in kv_nodes:
for gen_load in load_gen["ADD"][server]:
task = self.task.async_load_gen_docs(
self.cluster,
def_bucket,
gen_load,
"create",
0,
batch_size=50,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout)
self.task.jython_task_manager.get_task_result(task)
if len(task.fail.keys()) != 0:
self.log_failure("Some failures seen during doc_ops")
index_item_count["#primary"] += crud_batch_size
index_item_count["durable_add_aborts"] += crud_batch_size
expected_num_indexed["#primary"] += crud_batch_size
expected_num_indexed["durable_add_aborts"] += crud_batch_size
self.validate_indexed_doc_count(def_bucket, index_item_count)
for gen_load in load_gen["SET"][server]:
task = self.task.async_load_gen_docs(
self.cluster,
def_bucket,
gen_load,
"update",
0,
batch_size=50,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout)
self.task.jython_task_manager.get_task_result(task)
if len(task.fail.keys()) != 0:
self.log_failure("Some failures seen during doc_ops")
index_item_count["durable_add_aborts"] -= crud_batch_size
index_item_count["durable_set_aborts"] += crud_batch_size
expected_num_indexed["#primary"] += crud_batch_size
expected_num_indexed["durable_add_aborts"] += crud_batch_size
expected_num_indexed["durable_set_aborts"] += crud_batch_size
self.validate_indexed_doc_count(def_bucket, index_item_count)
self.log.info("Validate the mutated docs are taken into indexing")
self.validate_indexed_count_from_stats(def_bucket,
expected_num_indexed,
index_item_count)
self.validate_test_failure()
def setUp(self):
super(DurabilitySuccessTests, self).setUp()
self.durability_helper = DurabilityHelper(
self.log, len(self.cluster.nodes_in_cluster),
self.durability_level)
self.log.info("=== DurabilitySuccessTests setup complete ===")
def setUp(self):
super(RebalanceBaseTest, self).setUp()
self.rest = RestConnection(self.cluster.master)
self.doc_ops = self.input.param("doc_ops", "create")
self.key_size = self.input.param("key_size", 0)
self.zone = self.input.param("zone", 1)
self.replica_to_update = self.input.param("new_replica", None)
self.default_view_name = "default_view"
self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}"
self.default_view = View(self.default_view_name, self.defaul_map_func, None)
self.max_verify = self.input.param("max_verify", None)
self.std_vbucket_dist = self.input.param("std_vbucket_dist", None)
self.key = 'test_docs'.rjust(self.key_size, '0')
self.flusher_batch_split_trigger = self.input.param("flusher_batch_split_trigger", None)
self.items = self.num_items
node_ram_ratio = self.bucket_util.base_bucket_ratio(self.cluster.servers)
info = self.rest.get_nodes_self()
self.rest.init_cluster(username=self.cluster.master.rest_username,
password=self.cluster.master.rest_password)
self.rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved*node_ram_ratio))
self.check_temporary_failure_exception = False
nodes_init = self.cluster.servers[1:self.nodes_init] if self.nodes_init != 1 else []
if nodes_init:
result = self.task.rebalance([self.cluster.master], nodes_init, [])
self.assertTrue(result, "Initial rebalance failed")
self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init)
self.bucket_util.add_rbac_user()
if self.standard_buckets > 10:
self.bucket_util.change_max_buckets(self.standard_buckets)
self.create_buckets()
self.sleep(20)
if self.flusher_batch_split_trigger:
self.bucket_util.set_flusher_batch_split_trigger(self.cluster.master,
self.flusher_batch_split_trigger,
self.bucket_util.buckets)
self.gen_create = self.get_doc_generator(0, self.num_items)
if self.active_resident_threshold < 100:
self.check_temporary_failure_exception = True
if not self.atomicity:
tasks_info = self._load_all_buckets(self.cluster, self.gen_create, "create", 0)
self.log.info("Verifying num_items counts after doc_ops")
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
else:
self.transaction_commit = True
self._load_all_buckets_atomicty(self.gen_create, "create")
self.transaction_commit = self.input.param("transaction_commit", True)
# Initialize doc_generators
self.active_resident_threshold = 100
self.gen_create = None
self.gen_delete = None
self.gen_update = self.get_doc_generator(0, (self.items / 2))
self.durability_helper = DurabilityHelper(
self.log, len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to, persist_to=self.persist_to)
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.log.info("==========Finished rebalance base setup========")
def setUp(self):
super(CollectionBase, self).setUp()
self.log_setup_status("CollectionBase", "started")
self.key = 'test_collection'.rjust(self.key_size, '0')
self.simulate_error = self.input.param("simulate_error", None)
self.error_type = self.input.param("error_type", "memory")
self.doc_ops = self.input.param("doc_ops", None)
self.spec_name = self.input.param("bucket_spec",
"single_bucket.default")
self.over_ride_spec_params = \
self.input.param("override_spec_params", "").split(";")
self.action_phase = self.input.param("action_phase",
"before_default_load")
self.crud_batch_size = 100
self.num_nodes_affected = 1
if self.num_replicas > 1:
self.num_nodes_affected = 2
if self.doc_ops:
self.doc_ops = self.doc_ops.split(';')
self.durability_helper = DurabilityHelper(
self.log, len(self.cluster.nodes_in_cluster),
self.durability_level)
# Initialize cluster using given nodes
nodes_init = self.cluster.servers[1:self.nodes_init] \
if self.nodes_init != 1 else []
self.task.rebalance([self.cluster.master], nodes_init, [])
self.cluster.nodes_in_cluster.extend([self.cluster.master] +
nodes_init)
# Disable auto-failover to avoid failover of nodes
status = RestConnection(self.cluster.master) \
.update_autofailover_settings(False, 120, False)
self.assertTrue(status, msg="Failure during disabling auto-failover")
# Create bucket(s) and add rbac user
self.bucket_util.add_rbac_user()
buckets_spec = self.bucket_util.get_bucket_template_from_package(
self.spec_name)
doc_loading_spec = \
self.bucket_util.get_crud_template_from_package("initial_load")
# Process params to over_ride values if required
self.over_ride_template_params(buckets_spec)
self.over_ride_template_params(doc_loading_spec)
# MB-38438, adding CollectionNotFoundException in retry exception
doc_loading_spec[MetaCrudParams.RETRY_EXCEPTIONS].append(
SDKException.CollectionNotFoundException)
self.bucket_util.create_buckets_using_json_data(buckets_spec)
self.bucket_util.wait_for_collection_creation_to_complete()
# Init sdk_client_pool if not initialized before
if self.sdk_client_pool is None:
self.init_sdk_pool_object()
# Create clients in SDK client pool
self.log.info("Creating required SDK clients for client_pool")
bucket_count = len(self.bucket_util.buckets)
max_clients = self.task_manager.number_of_threads
clients_per_bucket = int(ceil(max_clients / bucket_count))
for bucket in self.bucket_util.buckets:
self.sdk_client_pool.create_clients(
bucket, [self.cluster.master],
clients_per_bucket,
compression_settings=self.sdk_compression)
# TODO: remove this once the bug is fixed
self.sleep(60, "MB-38497")
doc_loading_task = \
self.bucket_util.run_scenario_from_spec(
self.task,
self.cluster,
self.bucket_util.buckets,
doc_loading_spec,
mutation_num=0)
if doc_loading_task.result is False:
self.fail("Initial doc_loading failed")
self.cluster_util.print_cluster_stats()
# Verify initial doc load count
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.validate_docs_per_collections_all_buckets()
self.bucket_util.print_bucket_stats()
self.bucket_helper_obj = BucketHelper(self.cluster.master)
self.log_setup_status("CollectionBase", "complete")
def setUp(self):
super(RebalanceBaseTest, self).setUp()
self.rest = RestConnection(self.cluster.master)
self.doc_ops = self.input.param("doc_ops", "create")
self.key_size = self.input.param("key_size", 0)
self.zone = self.input.param("zone", 1)
self.replica_to_update = self.input.param("new_replica", None)
self.default_view_name = "default_view"
self.defaul_map_func = "function (doc) {\n emit(doc._id, doc);\n}"
self.default_view = View(self.default_view_name, self.defaul_map_func,
None)
self.max_verify = self.input.param("max_verify", None)
self.std_vbucket_dist = self.input.param("std_vbucket_dist", None)
self.flusher_total_batch_limit = self.input.param("flusher_total_batch_limit", None)
self.test_abort_snapshot = self.input.param("test_abort_snapshot",
False)
self.items = self.num_items
node_ram_ratio = self.bucket_util.base_bucket_ratio(self.cluster.servers)
info = self.rest.get_nodes_self()
self.rest.init_cluster(username=self.cluster.master.rest_username,
password=self.cluster.master.rest_password)
self.rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved*node_ram_ratio))
self.check_temporary_failure_exception = False
nodes_init = self.cluster.servers[1:self.nodes_init] if self.nodes_init != 1 else []
if nodes_init:
result = self.task.rebalance([self.cluster.master], nodes_init, [])
self.assertTrue(result, "Initial rebalance failed")
self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init)
self.check_replica = self.input.param("check_replica", False)
self.spec_name = self.input.param("bucket_spec", None)
# If buckets creation and initial data load is to be done by bucket_spec
if self.spec_name is not None:
self.log.info("Creating buckets from spec")
# Create bucket(s) and add rbac user
buckets_spec = self.bucket_util.get_bucket_template_from_package(
self.spec_name)
doc_loading_spec = \
self.bucket_util.get_crud_template_from_package("initial_load")
self.bucket_util.create_buckets_using_json_data(buckets_spec)
self.bucket_util.wait_for_collection_creation_to_complete()
# Create clients in SDK client pool
if self.sdk_client_pool:
self.log.info("Creating required SDK clients for client_pool")
bucket_count = len(self.bucket_util.buckets)
max_clients = self.task_manager.number_of_threads
clients_per_bucket = int(ceil(max_clients / bucket_count))
for bucket in self.bucket_util.buckets:
self.sdk_client_pool.create_clients(
bucket,
[self.cluster.master],
clients_per_bucket,
compression_settings=self.sdk_compression)
self.bucket_util.run_scenario_from_spec(self.task,
self.cluster,
self.bucket_util.buckets,
doc_loading_spec,
mutation_num=0)
self.bucket_util.add_rbac_user()
self.cluster_util.print_cluster_stats()
# Verify initial doc load count
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.validate_docs_per_collections_all_buckets()
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.bucket_helper_obj = BucketHelper(self.cluster.master)
self.log.info("==========Finished rebalance base setup========")
else:
self.bucket_util.add_rbac_user()
if self.standard_buckets > 10:
self.bucket_util.change_max_buckets(self.standard_buckets)
self.create_buckets(self.bucket_size)
# Create Scope/Collection based on inputs given
for bucket in self.bucket_util.buckets:
if self.scope_name != CbServer.default_scope:
self.scope_name = BucketUtils.get_random_name()
BucketUtils.create_scope(self.cluster.master,
bucket,
{"name": self.scope_name})
if self.collection_name != CbServer.default_collection:
self.collection_name = BucketUtils.get_random_name()
BucketUtils.create_collection(self.cluster.master,
bucket,
self.scope_name,
{"name": self.collection_name,
"num_items": self.num_items})
self.log.info("Bucket %s using scope::collection - '%s::%s'"
% (bucket.name,
self.scope_name,
self.collection_name))
# Update required num_items under default collection
bucket.scopes[self.scope_name] \
.collections[self.collection_name] \
.num_items = self.num_items
if self.flusher_total_batch_limit:
self.bucket_util.set_flusher_total_batch_limit(
self.cluster.master,
self.flusher_total_batch_limit,
self.bucket_util.buckets)
self.gen_create = self.get_doc_generator(0, self.num_items)
if self.active_resident_threshold < 100:
self.check_temporary_failure_exception = True
if not self.atomicity:
_ = self._load_all_buckets(self.cluster, self.gen_create,
"create", 0, batch_size=self.batch_size)
self.log.info("Verifying num_items counts after doc_ops")
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.validate_docs_per_collections_all_buckets(
timeout=120)
else:
self.transaction_commit = True
self._load_all_buckets_atomicty(self.gen_create, "create")
self.transaction_commit = self.input.param("transaction_commit",
True)
# Initialize doc_generators
self.active_resident_threshold = 100
self.gen_create = None
self.gen_delete = None
self.gen_update = self.get_doc_generator(0, (self.items / 2))
self.durability_helper = DurabilityHelper(
self.log, len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to, persist_to=self.persist_to)
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
self.log.info("==========Finished rebalance base setup========")
def test_fts_index_with_aborts(self):
"""
1. Create index (2i/view) on default bucket
2. Load multiple docs such that all sync_writes will be aborted
3. Verify nothing went into indexing
4. Load sync_write docs such that they are successful
5. Validate the mutated docs are taken into indexing
:return:
"""
self.key = "test_query_doc"
self.index_name = "fts_test_index"
self.sync_write_abort_pattern = self.input.param(
"sync_write_abort_pattern", "all_aborts")
self.create_index_during = self.input.param("create_index_during",
"before_doc_ops")
self.restServer = self.cluster_util.get_nodes_from_services_map(
cluster=self.cluster, service_type=CbServer.Services.FTS)
self.rest = RestConnection(self.restServer)
crud_batch_size = 1000
def_bucket = self.cluster.buckets[0]
kv_nodes = self.cluster_util.get_kv_nodes(self.cluster)
replica_vbs = dict()
verification_dict = dict()
index_item_count = dict()
expected_num_indexed = dict()
load_gen = dict()
load_gen["ADD"] = dict()
load_gen["SET"] = dict()
partial_aborts = ["initial_aborts", "aborts_at_end"]
durability_helper = DurabilityHelper(
self.log,
len(self.cluster.nodes_in_cluster),
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
if self.create_index_during == "before_doc_ops":
self.create_fts_indexes(def_bucket.name, self.index_name)
curr_items = self.bucket_util.get_bucket_current_item_count(
self.cluster, def_bucket)
if self.sync_write_abort_pattern in ["all_aborts", "initial_aborts"]:
self.bucket_util.flush_bucket(self.cluster, def_bucket)
self.num_items = 0
else:
self.num_items = curr_items
self.log.info("Disabling auto_failover to avoid node failures")
status = RestConnection(self.cluster.master) \
.update_autofailover_settings(False, 120, False)
self.assertTrue(status, msg="Failure during disabling auto-failover")
# Validate vbucket stats
verification_dict["ops_create"] = self.num_items
verification_dict["ops_update"] = 0
# verification_dict["ops_delete"] = 0
verification_dict["rollback_item_count"] = 0
verification_dict["sync_write_aborted_count"] = 0
verification_dict["sync_write_committed_count"] = 0
if self.create_index_during == "before_doc_ops":
self.validate_indexed_doc_count(self.index_name,
verification_dict["ops_create"])
self.log.info("Loading docs such that all sync_writes will be aborted")
for server in kv_nodes:
ssh_shell = RemoteMachineShellConnection(server)
cbstats = Cbstats(ssh_shell)
replica_vbs[server] = cbstats.vbucket_list(def_bucket.name,
"replica")
load_gen["ADD"][server] = list()
load_gen["ADD"][server].append(
doc_generator(self.key,
0,
crud_batch_size,
target_vbucket=replica_vbs[server],
mutation_type="ADD"))
if self.sync_write_abort_pattern in partial_aborts:
load_gen["ADD"][server].append(
doc_generator(self.key,
10000,
crud_batch_size,
target_vbucket=replica_vbs[server],
mutation_type="ADD"))
verification_dict["ops_create"] += crud_batch_size
verification_dict["sync_write_committed_count"] += \
crud_batch_size
task_success = self.bucket_util.load_durable_aborts(
ssh_shell, load_gen["ADD"][server], def_bucket,
self.durability_level, "create", self.sync_write_abort_pattern)
if not task_success:
self.log_failure("Failure during load_abort task")
verification_dict["sync_write_aborted_count"] += \
crud_batch_size
if self.create_index_during == "before_doc_ops":
self.validate_indexed_doc_count(
self.index_name, verification_dict["ops_create"])
load_gen["SET"][server] = list()
load_gen["SET"][server].append(
doc_generator(self.key,
0,
crud_batch_size,
target_vbucket=replica_vbs[server],
mutation_type="SET"))
if self.sync_write_abort_pattern in partial_aborts:
load_gen["SET"][server].append(
doc_generator(self.key,
10000,
crud_batch_size,
target_vbucket=replica_vbs[server],
mutation_type="SET"))
verification_dict["ops_update"] += crud_batch_size
verification_dict["sync_write_committed_count"] += \
crud_batch_size
verification_dict["sync_write_aborted_count"] += \
crud_batch_size
task_success = self.bucket_util.load_durable_aborts(
ssh_shell, load_gen["SET"][server], def_bucket,
self.durability_level, "update", self.sync_write_abort_pattern)
if not task_success:
self.log_failure("Failure during load_abort task")
ssh_shell.disconnect()
if self.create_index_during == "before_doc_ops":
self.validate_indexed_doc_count(
self.index_name, verification_dict["ops_create"])
failed = durability_helper.verify_vbucket_details_stats(
def_bucket,
kv_nodes,
vbuckets=self.cluster.vbuckets,
expected_val=verification_dict)
# if failed:
# self.sleep(6000)
# self.log_failure("Cbstat vbucket-details verification failed")
self.validate_test_failure()
if self.create_index_during == "after_doc_ops":
self.create_fts_indexes(def_bucket.name, self.index_name)
self.validate_indexed_doc_count(self.index_name,
verification_dict["ops_create"])
self.log.info("Verify aborts are not indexed")
self.validate_indexed_doc_count(self.index_name,
verification_dict["ops_create"])
for server in kv_nodes:
if self.sync_write_abort_pattern == "initial_aborts":
load_gen["ADD"][server] = load_gen["ADD"][server][:1]
load_gen["SET"][server] = load_gen["SET"][server][:1]
elif self.sync_write_abort_pattern == "aborts_at_end":
load_gen["ADD"][server] = load_gen["ADD"][server][-1:]
load_gen["SET"][server] = load_gen["SET"][server][-1:]
self.log.info("Load sync_write docs such that they are successful")
for server in kv_nodes:
for gen_load in load_gen["ADD"][server]:
task = self.task.async_load_gen_docs(
self.cluster,
def_bucket,
gen_load,
"create",
0,
batch_size=50,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout)
self.task.jython_task_manager.get_task_result(task)
if len(task.fail.keys()) != 0:
self.log_failure("Some failures seen during doc_ops")
verification_dict["ops_create"] += crud_batch_size
self.validate_indexed_doc_count(
self.index_name, verification_dict["ops_create"])
for gen_load in load_gen["SET"][server]:
task = self.task.async_load_gen_docs(
self.cluster,
def_bucket,
gen_load,
"update",
0,
batch_size=50,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout)
self.task.jython_task_manager.get_task_result(task)
if len(task.fail.keys()) != 0:
self.log_failure("Some failures seen during doc_ops")
verification_dict["ops_update"] += crud_batch_size
self.validate_indexed_doc_count(
self.index_name, verification_dict["ops_create"])
self.log.info("Validate the mutated docs are taken into indexing")
self.validate_indexed_doc_count(self.index_name,
verification_dict["ops_create"])
self.validate_test_failure()
