def wait_for_replication(servers, cluster_helper=None, timeout=600):
if cluster_helper is None:
cluster = ServerTasks()
else:
cluster = cluster_helper
tasks = []
rest = RestConnection(servers[0])
buckets = rest.get_buckets()
for server in servers:
for bucket in buckets:
for server_repl in list(set(servers) - set([server])):
tasks.append(
cluster.async_wait_for_stats(
[server], bucket, 'tap',
'eq_tapq:replication_ns_1@' + server_repl.ip +
':idle', '==', 'true'))
tasks.append(
cluster.async_wait_for_stats(
[server], bucket, 'tap',
'eq_tapq:replication_ns_1@' + server_repl.ip +
':backfill_completed', '==', 'true'))
try:
for task in tasks:
task.result(timeout)
finally:
if cluster_helper is None:
# stop all newly created task manager threads
cluster.shutdown()
return True
def setUp(self):
self.input = TestInputSingleton.input
# Framework specific parameters
self.log_level = self.input.param("log_level", "info").upper()
self.infra_log_level = self.input.param("infra_log_level",
"info").upper()
self.skip_setup_cleanup = self.input.param("skip_setup_cleanup", False)
self.tear_down_while_setup = self.input.param("tear_down_while_setup",
True)
self.test_timeout = self.input.param("test_timeout", 3600)
self.thread_to_use = self.input.param("threads_to_use", 10)
self.case_number = self.input.param("case_number", 0)
# End of framework parameters
# Cluster level info settings
self.log_info = self.input.param("log_info", None)
self.log_location = self.input.param("log_location", None)
self.stat_info = self.input.param("stat_info", None)
self.port = self.input.param("port", None)
self.port_info = self.input.param("port_info", None)
self.servers = self.input.servers
self.__cb_clusters = []
self.num_servers = self.input.param("servers", len(self.servers))
self.primary_index_created = False
self.index_quota_percent = self.input.param("index_quota_percent",
None)
self.gsi_type = self.input.param("gsi_type", 'plasma')
# CBAS setting
self.jre_path = self.input.param("jre_path", None)
# End of cluster info parameters
# Bucket specific params
self.bucket_type = self.input.param("bucket_type",
Bucket.bucket_type.MEMBASE)
self.bucket_size = self.input.param("bucket_size", None)
self.bucket_lww = self.input.param("lww", True)
self.standard_buckets = self.input.param("standard_buckets", 1)
if self.standard_buckets > 10:
self.bucket_util.change_max_buckets(self.standard_buckets)
self.vbuckets = self.input.param("vbuckets", 1024)
self.num_replicas = self.input.param("replicas", 1)
self.active_resident_threshold = int(
self.input.param("active_resident_threshold", 100))
self.compression_mode = self.input.param("compression_mode", 'passive')
# End of bucket parameters
# Doc specific params
self.key_size = self.input.param("key_size", 0)
self.doc_size = self.input.param("doc_size", 10)
self.sub_doc_size = self.input.param("sub_doc_size", 10)
self.doc_type = self.input.param("doc_type", "json")
self.num_items = self.input.param("num_items", 100000)
self.target_vbucket = self.input.param("target_vbucket", None)
self.maxttl = self.input.param("maxttl", 0)
# End of doc specific parameters
# Transactions parameters
self.transaction_timeout = self.input.param("transaction_timeout", 100)
self.transaction_commit = self.input.param("transaction_commit", True)
self.update_count = self.input.param("update_count", 1)
self.sync = self.input.param("sync", True)
self.default_bucket = self.input.param("default_bucket", True)
self.num_buckets = self.input.param("num_buckets", 0)
self.atomicity = self.input.param("atomicity", False)
# end of transaction parameters
# Client specific params
self.sdk_client_type = self.input.param("sdk_client_type", "java")
self.sdk_compression = self.input.param("sdk_compression", True)
self.replicate_to = self.input.param("replicate_to", 0)
self.persist_to = self.input.param("persist_to", 0)
self.sdk_retries = self.input.param("sdk_retries", 5)
self.sdk_timeout = self.input.param("sdk_timeout", 5)
self.durability_level = self.input.param("durability", "")
# Doc Loader Params
self.process_concurrency = self.input.param("process_concurrency", 8)
self.batch_size = self.input.param("batch_size", 20)
self.ryow = self.input.param("ryow", False)
self.check_persistence = self.input.param("check_persistence", False)
# End of client specific parameters
# initial number of items in the cluster
self.services_init = self.input.param("services_init", None)
self.nodes_init = self.input.param("nodes_init", 1)
self.nodes_in = self.input.param("nodes_in", 1)
self.nodes_out = self.input.param("nodes_out", 1)
self.services_in = self.input.param("services_in", None)
self.forceEject = self.input.param("forceEject", False)
self.wait_timeout = self.input.param("wait_timeout", 60)
self.dgm_run = self.input.param("dgm_run", False)
self.verify_unacked_bytes = self.input.param("verify_unacked_bytes",
False)
self.disabled_consistent_view = self.input.param(
"disabled_consistent_view", None)
self.rebalanceIndexWaitingDisabled = self.input.param(
"rebalanceIndexWaitingDisabled", None)
self.rebalanceIndexPausingDisabled = self.input.param(
"rebalanceIndexPausingDisabled", None)
self.maxParallelIndexers = self.input.param("maxParallelIndexers",
None)
self.maxParallelReplicaIndexers = self.input.param(
"maxParallelReplicaIndexers", None)
self.quota_percent = self.input.param("quota_percent", None)
if not hasattr(self, 'skip_buckets_handle'):
self.skip_buckets_handle = self.input.param(
"skip_buckets_handle", False)
# Initiate logging variables
self.log = logging.getLogger("test")
self.infra_log = logging.getLogger("infra")
# Configure loggers
self.log.setLevel(self.log_level)
self.infra_log.setLevel(self.infra_log_level)
# Support lib objects for testcase execution
self.task_manager = TaskManager(self.thread_to_use)
self.task = ServerTasks(self.task_manager)
# End of library object creation
self.cleanup = False
self.nonroot = False
self.test_failure = None
self.__log_setup_status("started")
if len(self.input.clusters) > 1:
# Multi cluster setup
counter = 1
for _, nodes in self.input.clusters.iteritems():
self.__cb_clusters.append(
CBCluster(name="C%s" % counter, servers=nodes))
counter += 1
else:
# Single cluster
self.cluster = CBCluster(servers=self.servers)
self.__cb_clusters.append(self.cluster)
self.cluster_util = ClusterUtils(self.cluster, self.task_manager)
self.bucket_util = BucketUtils(self.cluster, self.cluster_util,
self.task)
for cluster in self.__cb_clusters:
shell = RemoteMachineShellConnection(cluster.master)
self.os_info = shell.extract_remote_info().type.lower()
if self.os_info != 'windows':
if cluster.master.ssh_username != "root":
self.nonroot = True
shell.disconnect()
break
""" some tests need to bypass checking cb server at set up
to run installation """
self.skip_init_check_cbserver = self.input.param(
"skip_init_check_cbserver", False)
try:
if self.skip_setup_cleanup:
self.buckets = self.bucket_util.get_all_buckets()
return
if not self.skip_init_check_cbserver:
for cluster in self.__cb_clusters:
self.cb_version = None
if RestHelper(RestConnection(
cluster.master)).is_ns_server_running():
"""
Since every new couchbase version, there will be new
features that test code won't work on previous release.
So we need to get couchbase version to filter out
those tests.
"""
self.cb_version = RestConnection(
cluster.master).get_nodes_version()
else:
self.log.debug("couchbase server does not run yet")
# We stopped supporting TAP protocol since 3.x and 3.x support also has stopped
self.protocol = "dcp"
self.services_map = None
self.__log_setup_status("started")
for cluster in self.__cb_clusters:
if not self.skip_buckets_handle and not self.skip_init_check_cbserver:
self.log.debug("Cleaning up cluster")
cluster_util = ClusterUtils(cluster, self.task_manager)
bucket_util = BucketUtils(cluster, cluster_util, self.task)
cluster_util.cluster_cleanup(bucket_util)
# avoid any cluster operations in setup for new upgrade
# & upgradeXDCR tests
if str(self.__class__).find('newupgradetests') != -1 or \
str(self.__class__).find('upgradeXDCR') != -1 or \
str(self.__class__).find('Upgrade_EpTests') != -1 or \
hasattr(self, 'skip_buckets_handle') and \
self.skip_buckets_handle:
self.log.warning(
"any cluster operation in setup will be skipped")
self.primary_index_created = True
self.__log_setup_status("finished")
return
# avoid clean up if the previous test has been tear down
if self.case_number == 1 or self.case_number > 1000:
if self.case_number > 1000:
self.log.warn(
"TearDown for previous test failed. will retry..")
self.case_number -= 1000
self.cleanup = True
if not self.skip_init_check_cbserver:
self.tearDownEverything()
self.tear_down_while_setup = False
if not self.skip_init_check_cbserver:
for cluster in self.__cb_clusters:
self.log.info("Initializing cluster")
cluster_util = ClusterUtils(cluster, self.task_manager)
# self.cluster_util.reset_cluster()
master_services = cluster_util.get_services(
cluster.servers[:1], self.services_init, start_node=0)
if master_services is not None:
master_services = master_services[0].split(",")
self.quota = self._initialize_nodes(
self.task,
cluster,
self.disabled_consistent_view,
self.rebalanceIndexWaitingDisabled,
self.rebalanceIndexPausingDisabled,
self.maxParallelIndexers,
self.maxParallelReplicaIndexers,
self.port,
self.quota_percent,
services=master_services)
cluster_util.change_env_variables()
cluster_util.change_checkpoint_params()
#cluster_util.add_all_nodes_then_rebalance(cluster.servers[1:])
self.log.info("{0} initialized".format(cluster))
else:
self.quota = ""
for cluster in self.__cb_clusters:
cluster_util = ClusterUtils(cluster, self.task_manager)
if self.log_info:
cluster_util.change_log_info()
if self.log_location:
cluster_util.change_log_location()
if self.stat_info:
cluster_util.change_stat_info()
if self.port_info:
cluster_util.change_port_info()
if self.port:
self.port = str(self.port)
self.__log_setup_status("finished")
if not self.skip_init_check_cbserver:
self.__log("started")
self.sleep(5)
except Exception, e:
traceback.print_exc()
self.task.shutdown(force=True)
self.fail(e)
def setUp(self):
self.log = logger.Logger.get_logger()
self.input = TestInputSingleton.input
self.primary_index_created = False
self.use_sdk_client = self.input.param("use_sdk_client", False)
if self.input.param("log_level", None):
log.setLevel(level=0)
for hd in log.handlers:
if str(hd.__class__).find('FileHandler') != -1:
hd.setLevel(level=logging.DEBUG)
else:
hd.setLevel(level=getattr(
logging, self.input.param("log_level", None)))
self.servers = self.input.servers
self.buckets = []
self.case_number = self.input.param("case_number", 0)
self.thread_to_use = self.input.param("threads_to_use", 10)
self.cluster = CBCluster(servers=self.input.servers)
self.task_manager = TaskManager(self.thread_to_use)
self.cluster_util = cluster_utils(self.cluster, self.task_manager)
self.bucket_util = bucket_utils(self.cluster, self.task_manager,
self.cluster_util)
self.task = ServerTasks(self.task_manager)
self.cleanup = False
self.nonroot = False
shell = RemoteMachineShellConnection(self.cluster.master)
self.os_info = shell.extract_remote_info().type.lower()
if self.os_info != 'windows':
if self.cluster.master.ssh_username != "root":
self.nonroot = True
shell.disconnect()
""" some tests need to bypass checking cb server at set up
to run installation """
self.skip_init_check_cbserver = self.input.param(
"skip_init_check_cbserver", False)
try:
self.vbuckets = self.input.param("vbuckets", 1024)
self.skip_setup_cleanup = self.input.param("skip_setup_cleanup",
False)
self.index_quota_percent = self.input.param(
"index_quota_percent", None)
self.num_servers = self.input.param("servers",
len(self.cluster.servers))
# initial number of items in the cluster
self.services_init = self.input.param("services_init", None)
self.nodes_init = self.input.param("nodes_init", 1)
self.nodes_in = self.input.param("nodes_in", 1)
self.nodes_out = self.input.param("nodes_out", 1)
self.services_in = self.input.param("services_in", None)
self.forceEject = self.input.param("forceEject", False)
self.num_items = self.input.param("num_items", 100000)
self.num_replicas = self.input.param("replicas", 1)
self.value_size = self.input.param("value_size", 1)
self.wait_timeout = self.input.param("wait_timeout", 60)
self.dgm_run = self.input.param("dgm_run", False)
self.active_resident_threshold = int(
self.input.param("active_resident_threshold", 0))
self.verify_unacked_bytes = self.input.param(
"verify_unacked_bytes", False)
self.force_kill_memcached = TestInputSingleton.input.param(
'force_kill_memcached', False)
self.disabled_consistent_view = self.input.param(
"disabled_consistent_view", None)
self.rebalanceIndexWaitingDisabled = self.input.param(
"rebalanceIndexWaitingDisabled", None)
self.rebalanceIndexPausingDisabled = self.input.param(
"rebalanceIndexPausingDisabled", None)
self.maxParallelIndexers = self.input.param(
"maxParallelIndexers", None)
self.maxParallelReplicaIndexers = self.input.param(
"maxParallelReplicaIndexers", None)
self.quota_percent = self.input.param("quota_percent", None)
self.port = None
self.log_info = self.input.param("log_info", None)
self.log_location = self.input.param("log_location", None)
self.stat_info = self.input.param("stat_info", None)
self.port_info = self.input.param("port_info", None)
if not hasattr(self, 'skip_buckets_handle'):
self.skip_buckets_handle = self.input.param(
"skip_buckets_handle", False)
self.test_timeout = self.input.param(
"test_timeout", 3600) # kill hang test and jump to next one.
self.gsi_type = self.input.param("gsi_type", 'plasma')
self.compression_mode = self.input.param("compression_mode",
'passive')
self.sdk_compression = self.input.param("sdk_compression", True)
self.replicate_to = self.input.param("replicate_to", 0)
self.persist_to = self.input.param("persist_to", 0)
#jre-path for cbas
self.jre_path = self.input.param("jre_path", None)
# end of bucket parameters spot (this is ongoing)
if self.skip_setup_cleanup:
self.buckets = self.bucket_util.get_all_buckets()
return
if not self.skip_init_check_cbserver:
self.cb_version = None
if RestHelper(RestConnection(
self.cluster.master)).is_ns_server_running():
""" since every new couchbase version, there will be new features
that test code will not work on previous release. So we need
to get couchbase version to filter out those tests. """
self.cb_version = RestConnection(
self.cluster.master).get_nodes_version()
else:
log.info("couchbase server does not run yet")
self.protocol = self.cluster_util.get_protocol_type()
self.services_map = None
log.info("============== basetestcase setup was started for test #{0} {1}==============" \
.format(self.case_number, self._testMethodName))
if not self.skip_buckets_handle and not self.skip_init_check_cbserver:
self.cluster_util.cluster_cleanup(self.bucket_util)
# avoid any cluster operations in setup for new upgrade
# & upgradeXDCR tests
if str(self.__class__).find('newupgradetests') != -1 or \
str(self.__class__).find('upgradeXDCR') != -1 or \
str(self.__class__).find('Upgrade_EpTests') != -1 or \
hasattr(self, 'skip_buckets_handle') and \
self.skip_buckets_handle:
log.info("any cluster operation in setup will be skipped")
self.primary_index_created = True
log.info("============== basetestcase setup was finished for test #{0} {1} ==============" \
.format(self.case_number, self._testMethodName))
return
# avoid clean up if the previous test has been tear down
if self.case_number == 1 or self.case_number > 1000:
if self.case_number > 1000:
log.warn(
"teardDown for previous test failed. will retry..")
self.case_number -= 1000
self.cleanup = True
if not self.skip_init_check_cbserver:
self.tearDownEverything()
self.task = ServerTasks(self.task_manager)
if not self.skip_init_check_cbserver:
log.info("initializing cluster")
# self.cluster_util.reset_cluster()
master_services = self.cluster_util.get_services(self.servers[:1], \
self.services_init, \
start_node=0)
if master_services != None:
master_services = master_services[0].split(",")
self.quota = self._initialize_nodes(self.task, self.cluster.servers, \
self.disabled_consistent_view, \
self.rebalanceIndexWaitingDisabled, \
self.rebalanceIndexPausingDisabled, \
self.maxParallelIndexers, \
self.maxParallelReplicaIndexers, \
self.port, \
self.quota_percent, \
services=master_services)
self.cluster_util.change_env_variables()
self.cluster_util.change_checkpoint_params()
log.info("done initializing cluster")
else:
self.quota = ""
if self.input.param("log_info", None):
self.cluster_util.change_log_info()
if self.input.param("log_location", None):
self.cluster_util.change_log_location()
if self.input.param("stat_info", None):
self.cluster_util.change_stat_info()
if self.input.param("port_info", None):
self.cluster_util.change_port_info()
if self.input.param("port", None):
self.port = str(self.input.param("port", None))
log.info("============== basetestcase setup was finished for test #{0} {1} ==============" \
.format(self.case_number, self._testMethodName))
if not self.skip_init_check_cbserver:
self._log_start()
self.sleep(5)
except Exception, e:
traceback.print_exc()
self.task.shutdown(force=True)
self.fail(e)
def setUp(self):
self.input = TestInputSingleton.input
# Framework specific parameters
self.log_level = self.input.param("log_level", "info").upper()
self.infra_log_level = self.input.param("infra_log_level",
"error").upper()
self.skip_setup_cleanup = self.input.param("skip_setup_cleanup", False)
self.tear_down_while_setup = self.input.param("tear_down_while_setup",
True)
self.test_timeout = self.input.param("test_timeout", 3600)
self.thread_to_use = self.input.param("threads_to_use", 30)
self.case_number = self.input.param("case_number", 0)
# End of framework parameters
# Cluster level info settings
self.log_info = self.input.param("log_info", None)
self.log_location = self.input.param("log_location", None)
self.stat_info = self.input.param("stat_info", None)
self.port = self.input.param("port", None)
self.port_info = self.input.param("port_info", None)
self.servers = self.input.servers
self.cb_clusters = OrderedDict()
self.num_servers = self.input.param("servers", len(self.servers))
self.primary_index_created = False
self.index_quota_percent = self.input.param("index_quota_percent",
None)
self.gsi_type = self.input.param("gsi_type", 'plasma')
# CBAS setting
self.jre_path = self.input.param("jre_path", None)
self.enable_dp = self.input.param("enable_dp", False)
# End of cluster info parameters
# Bucket specific params
self.bucket_type = self.input.param("bucket_type", Bucket.Type.MEMBASE)
self.bucket_ttl = self.input.param("bucket_ttl", 0)
self.bucket_size = self.input.param("bucket_size", None)
self.bucket_conflict_resolution_type = \
self.input.param("bucket_conflict_resolution",
Bucket.ConflictResolution.SEQ_NO)
self.bucket_replica_index = self.input.param("bucket_replica_index", 1)
self.bucket_eviction_policy = \
self.input.param("bucket_eviction_policy",
Bucket.EvictionPolicy.VALUE_ONLY)
self.flush_enabled = self.input.param("flushEnabled",
Bucket.FlushBucket.DISABLED)
self.bucket_time_sync = self.input.param("bucket_time_sync", False)
self.standard_buckets = self.input.param("standard_buckets", 1)
self.num_replicas = self.input.param("replicas", Bucket.ReplicaNum.ONE)
self.active_resident_threshold = \
int(self.input.param("active_resident_threshold", 100))
self.compression_mode = \
self.input.param("compression_mode",
Bucket.CompressionMode.PASSIVE)
self.bucket_storage = \
self.input.param("bucket_storage",
Bucket.StorageBackend.couchstore)
if self.bucket_storage == Bucket.StorageBackend.magma:
self.bucket_eviction_policy = Bucket.EvictionPolicy.FULL_EVICTION
self.scope_name = self.input.param("scope", CbServer.default_scope)
self.collection_name = self.input.param("collection",
CbServer.default_collection)
self.bucket_durability_level = self.input.param(
"bucket_durability", Bucket.DurabilityLevel.NONE).upper()
self.bucket_purge_interval = self.input.param("bucket_purge_interval",
1)
self.bucket_durability_level = \
BucketDurability[self.bucket_durability_level]
# End of bucket parameters
# Doc specific params
self.key = self.input.param("key", "test_docs")
self.key_size = self.input.param("key_size", 8)
self.doc_size = self.input.param("doc_size", 256)
self.sub_doc_size = self.input.param("sub_doc_size", 10)
self.doc_type = self.input.param("doc_type", "json")
self.num_items = self.input.param("num_items", 100000)
self.target_vbucket = self.input.param("target_vbucket", None)
self.maxttl = self.input.param("maxttl", 0)
self.random_exp = self.input.param("random_exp", False)
self.randomize_doc_size = self.input.param("randomize_doc_size", False)
self.randomize_value = self.input.param("randomize_value", False)
self.rev_write = self.input.param("rev_write", False)
self.rev_read = self.input.param("rev_read", False)
self.rev_update = self.input.param("rev_update", False)
self.rev_del = self.input.param("rev_del", False)
self.random_key = self.input.param("random_key", False)
self.mix_key_size = self.input.param("mix_key_size", False)
# End of doc specific parameters
# Transactions parameters
self.transaction_timeout = self.input.param("transaction_timeout", 100)
self.transaction_commit = self.input.param("transaction_commit", True)
self.update_count = self.input.param("update_count", 1)
self.sync = self.input.param("sync", True)
self.default_bucket = self.input.param("default_bucket", True)
self.num_buckets = self.input.param("num_buckets", 0)
self.atomicity = self.input.param("atomicity", False)
self.defer = self.input.param("defer", False)
# end of transaction parameters
# Client specific params
self.sdk_client_type = self.input.param("sdk_client_type", "java")
self.replicate_to = self.input.param("replicate_to", 0)
self.persist_to = self.input.param("persist_to", 0)
self.sdk_retries = self.input.param("sdk_retries", 5)
self.sdk_timeout = self.input.param("sdk_timeout", 5)
self.time_unit = self.input.param("time_unit", "seconds")
self.durability_level = self.input.param("durability", "").upper()
self.sdk_client_pool = self.input.param("sdk_client_pool", None)
self.sdk_pool_capacity = self.input.param("sdk_pool_capacity", 1)
# Client compression settings
self.sdk_compression = self.input.param("sdk_compression", None)
compression_min_ratio = self.input.param("min_ratio", None)
compression_min_size = self.input.param("min_size", None)
if type(self.sdk_compression) is bool:
self.sdk_compression = {"enabled": self.sdk_compression}
if compression_min_size:
self.sdk_compression["minSize"] = compression_min_size
if compression_min_ratio:
self.sdk_compression["minRatio"] = compression_min_ratio
# Doc Loader Params
self.process_concurrency = self.input.param("process_concurrency", 20)
self.batch_size = self.input.param("batch_size", 2000)
self.dgm_batch = self.input.param("dgm_batch", 5000)
self.ryow = self.input.param("ryow", False)
self.check_persistence = self.input.param("check_persistence", False)
# End of client specific parameters
# initial number of items in the cluster
self.services_init = self.input.param("services_init", None)
self.nodes_init = self.input.param("nodes_init", 1)
self.nodes_in = self.input.param("nodes_in", 1)
self.nodes_out = self.input.param("nodes_out", 1)
self.services_in = self.input.param("services_in", None)
self.forceEject = self.input.param("forceEject", False)
self.wait_timeout = self.input.param("wait_timeout", 120)
self.verify_unacked_bytes = \
self.input.param("verify_unacked_bytes", False)
self.disabled_consistent_view = \
self.input.param("disabled_consistent_view", None)
self.rebalanceIndexWaitingDisabled = \
self.input.param("rebalanceIndexWaitingDisabled", None)
self.rebalanceIndexPausingDisabled = \
self.input.param("rebalanceIndexPausingDisabled", None)
self.maxParallelIndexers = \
self.input.param("maxParallelIndexers", None)
self.maxParallelReplicaIndexers = \
self.input.param("maxParallelReplicaIndexers", None)
self.quota_percent = self.input.param("quota_percent", 90)
self.skip_buckets_handle = self.input.param("skip_buckets_handle",
False)
# SDKClientPool object for creating generic clients across tasks
if self.sdk_client_pool is True:
self.init_sdk_pool_object()
# Initiate logging variables
self.log = logger.get("test")
self.infra_log = logger.get("infra")
self.cleanup_pcaps()
self.collect_pcaps = self.input.param("collect_pcaps", False)
if self.collect_pcaps:
self.start_collect_pcaps()
# variable for log collection using cbCollect
self.get_cbcollect_info = self.input.param("get-cbcollect-info", False)
# Variable for initializing the current (start of test) timestamp
self.start_timestamp = datetime.now()
'''
Be careful while using this flag.
This is only and only for stand-alone tests.
During bugs reproductions, when a crash is seen
stop_server_on_crash will stop the server
so that we can collect data/logs/dumps at the right time
'''
self.stop_server_on_crash = self.input.param("stop_server_on_crash",
False)
self.collect_data = self.input.param("collect_data", False)
# Configure loggers
self.log.setLevel(self.log_level)
self.infra_log.setLevel(self.infra_log_level)
# Support lib objects for testcase execution
self.task_manager = TaskManager(self.thread_to_use)
self.task = ServerTasks(self.task_manager)
# End of library object creation
self.sleep = sleep
self.cleanup = False
self.nonroot = False
self.test_failure = None
self.crash_warning = self.input.param("crash_warning", False)
self.summary = TestSummary(self.log)
# Populate memcached_port in case of cluster_run
cluster_run_base_port = ClusterRun.port
if int(self.input.servers[0].port) == ClusterRun.port:
for server in self.input.servers:
server.port = cluster_run_base_port
cluster_run_base_port += 1
# If not defined in node.ini under 'memcached_port' section
if server.memcached_port is CbServer.memcached_port:
server.memcached_port = \
ClusterRun.memcached_port \
+ (2 * (int(server.port) - ClusterRun.port))
self.log_setup_status(self.__class__.__name__, "started")
cluster_name_format = "C%s"
default_cluster_index = counter_index = 1
if len(self.input.clusters) > 1:
# Multi cluster setup
for _, nodes in self.input.clusters.iteritems():
cluster_name = cluster_name_format % counter_index
tem_cluster = CBCluster(name=cluster_name, servers=nodes)
self.cb_clusters[cluster_name] = tem_cluster
counter_index += 1
else:
# Single cluster
cluster_name = cluster_name_format % counter_index
self.cb_clusters[cluster_name] = CBCluster(name=cluster_name,
servers=self.servers)
# Initialize self.cluster with first available cluster as default
self.cluster = self.cb_clusters[cluster_name_format %
default_cluster_index]
self.cluster_util = ClusterUtils(self.cluster, self.task_manager)
self.bucket_util = BucketUtils(self.cluster_util, self.task)
if self.standard_buckets > 10:
self.bucket_util.change_max_buckets(self.cluster.master,
self.standard_buckets)
for cluster_name, cluster in self.cb_clusters.items():
shell = RemoteMachineShellConnection(cluster.master)
self.os_info = shell.extract_remote_info().type.lower()
if self.os_info != 'windows':
if cluster.master.ssh_username != "root":
self.nonroot = True
shell.disconnect()
break
shell.disconnect()
""" some tests need to bypass checking cb server at set up
to run installation """
self.skip_init_check_cbserver = \
self.input.param("skip_init_check_cbserver", False)
try:
if self.skip_setup_cleanup:
self.cluster.buckets = self.bucket_util.get_all_buckets(
self.cluster)
return
self.services_map = None
self.log_setup_status("BaseTestCase", "started")
for cluster_name, cluster in self.cb_clusters.items():
if not self.skip_buckets_handle \
and not self.skip_init_check_cbserver:
self.log.debug("Cleaning up cluster")
cluster_util = ClusterUtils(cluster, self.task_manager)
bucket_util = BucketUtils(cluster_util, self.task)
cluster_util.cluster_cleanup(bucket_util)
# Avoid cluster operations in setup for new upgrade / upgradeXDCR
if str(self.__class__).find('newupgradetests') != -1 or \
str(self.__class__).find('upgradeXDCR') != -1 or \
str(self.__class__).find('Upgrade_EpTests') != -1 or \
self.skip_buckets_handle:
self.log.warning("Cluster operation in setup will be skipped")
self.primary_index_created = True
self.log_setup_status("BaseTestCase", "finished")
return
# avoid clean up if the previous test has been tear down
if self.case_number == 1 or self.case_number > 1000:
if self.case_number > 1000:
self.log.warn("TearDown for prev test failed. Will retry")
self.case_number -= 1000
self.cleanup = True
if not self.skip_init_check_cbserver:
self.tearDownEverything()
self.tear_down_while_setup = False
if not self.skip_init_check_cbserver:
for cluster_name, cluster in self.cb_clusters.items():
self.log.info("Initializing cluster")
cluster_util = ClusterUtils(cluster, self.task_manager)
cluster_util.reset_cluster()
master_services = cluster_util.get_services(
cluster.servers[:1], self.services_init, start_node=0)
if master_services is not None:
master_services = master_services[0].split(",")
self.quota = self._initialize_nodes(
self.task,
cluster,
self.disabled_consistent_view,
self.rebalanceIndexWaitingDisabled,
self.rebalanceIndexPausingDisabled,
self.maxParallelIndexers,
self.maxParallelReplicaIndexers,
self.port,
self.quota_percent,
services=master_services)
cluster_util.change_env_variables()
cluster_util.change_checkpoint_params()
self.log.info("{0} initialized".format(cluster))
else:
self.quota = ""
# Enable dp_version since we need collections enabled
if self.enable_dp:
for server in self.cluster.servers:
shell_conn = RemoteMachineShellConnection(server)
cb_cli = CbCli(shell_conn)
cb_cli.enable_dp()
shell_conn.disconnect()
for cluster_name, cluster in self.cb_clusters.items():
cluster_util = ClusterUtils(cluster, self.task_manager)
if self.log_info:
cluster_util.change_log_info()
if self.log_location:
cluster_util.change_log_location()
if self.stat_info:
cluster_util.change_stat_info()
if self.port_info:
cluster_util.change_port_info()
if self.port:
self.port = str(self.port)
self.log_setup_status("BaseTestCase", "finished")
if not self.skip_init_check_cbserver:
self.__log("started")
except Exception as e:
traceback.print_exc()
self.task.shutdown(force=True)
self.fail(e)
def setUp(self):
self.failover_util = failover_utils()
self.node_util = node_utils()
self.views_util = views_utils()
self.log = logger.Logger.get_logger()
self.input = TestInputSingleton.input
self.primary_index_created = False
self.use_sdk_client = self.input.param("use_sdk_client", False)
if self.input.param("log_level", None):
log.setLevel(level=0)
for hd in log.handlers:
if str(hd.__class__).find('FileHandler') != -1:
hd.setLevel(level=logging.DEBUG)
else:
hd.setLevel(level=getattr(logging, self.input.param("log_level", None)))
self.servers = self.input.servers
if str(self.__class__).find('moxitests') != -1:
self.moxi_server = self.input.moxis[0]
self.servers = [server for server in self.servers
if server.ip != self.moxi_server.ip]
self.buckets = []
self.cluster = CBCluster(servers=self.input.servers)
self.bucket_util = bucket_utils(self.cluster)
self.cluster_util = cluster_utils(self.cluster)
self.task = ServerTasks()
self.pre_warmup_stats = {}
self.cleanup = False
self.nonroot = False
shell = RemoteMachineShellConnection(self.cluster.master)
self.os_info = shell.extract_remote_info().type.lower()
if self.os_info != 'windows':
if self.cluster.master.ssh_username != "root":
self.nonroot = True
shell.disconnect()
""" some tests need to bypass checking cb server at set up
to run installation """
self.skip_init_check_cbserver = \
self.input.param("skip_init_check_cbserver", False)
self.data_collector = DataCollector()
self.data_analyzer = DataAnalyzer()
self.result_analyzer = DataAnalysisResultAnalyzer()
try:
self.skip_setup_cleanup = self.input.param("skip_setup_cleanup", False)
self.index_quota_percent = self.input.param("index_quota_percent", None)
self.targetIndexManager = self.input.param("targetIndexManager", False)
self.targetMaster = self.input.param("targetMaster", False)
self.reset_services = self.input.param("reset_services", False)
self.auth_mech = self.input.param("auth_mech", "PLAIN")
self.wait_timeout = self.input.param("wait_timeout", 60)
# number of case that is performed from testrunner( increment each time)
self.case_number = self.input.param("case_number", 0)
self.default_bucket = self.input.param("default_bucket", True)
self.parallelism = self.input.param("parallelism", False)
self.verify_unacked_bytes = self.input.param("verify_unacked_bytes", False)
self.enable_flow_control = self.input.param("enable_flow_control", False)
self.num_servers = self.input.param("servers", len(self.cluster.servers))
# initial number of items in the cluster
self.nodes_init = self.input.param("nodes_init", 1)
self.nodes_in = self.input.param("nodes_in", 1)
self.nodes_out = self.input.param("nodes_out", 1)
self.services_init = self.input.param("services_init", None)
self.services_in = self.input.param("services_in", None)
self.forceEject = self.input.param("forceEject", False)
self.force_kill_memcached = TestInputSingleton.input.param('force_kill_memcached', False)
self.num_items = self.input.param("items", 1000)
self.value_size = self.input.param("value_size", 512)
self.dgm_run = self.input.param("dgm_run", False)
self.active_resident_threshold = int(self.input.param("active_resident_threshold", 0))
# max items number to verify in ValidateDataTask, None - verify all
self.max_verify = self.input.param("max_verify", None)
# we don't change consistent_view on server by default
self.disabled_consistent_view = self.input.param("disabled_consistent_view", None)
self.rebalanceIndexWaitingDisabled = self.input.param("rebalanceIndexWaitingDisabled", None)
self.rebalanceIndexPausingDisabled = self.input.param("rebalanceIndexPausingDisabled", None)
self.maxParallelIndexers = self.input.param("maxParallelIndexers", None)
self.maxParallelReplicaIndexers = self.input.param("maxParallelReplicaIndexers", None)
self.quota_percent = self.input.param("quota_percent", None)
self.port = None
self.log_message = self.input.param("log_message", None)
self.log_info = self.input.param("log_info", None)
self.log_location = self.input.param("log_location", None)
self.stat_info = self.input.param("stat_info", None)
self.port_info = self.input.param("port_info", None)
if not hasattr(self, 'skip_buckets_handle'):
self.skip_buckets_handle = self.input.param("skip_buckets_handle", False)
self.nodes_out_dist = self.input.param("nodes_out_dist", None)
self.absolute_path = self.input.param("absolute_path", True)
self.test_timeout = self.input.param("test_timeout", 3600) # kill hang test and jump to next one.
self.enable_bloom_filter = self.input.param("enable_bloom_filter", False)
# self.enable_time_sync = self.input.param("enable_time_sync", False)
self.gsi_type = self.input.param("gsi_type", 'plasma')
#jre-path for cbas
self.jre_path=self.input.param("jre_path",None)
# end of bucket parameters spot (this is ongoing)
if self.skip_setup_cleanup:
self.buckets = BucketHelper(self.master).get_buckets()
return
if not self.skip_init_check_cbserver:
self.cb_version = None
if RestHelper(RestConnection(self.cluster.master)).is_ns_server_running():
""" since every new couchbase version, there will be new features
that test code will not work on previous release. So we need
to get couchbase version to filter out those tests. """
self.cb_version = RestConnection(self.cluster.master).get_nodes_version()
else:
log.info("couchbase server does not run yet")
self.protocol = self.cluster_util.get_protocol_type()
self.services_map = None
log.info("============== basetestcase setup was started for test #{0} {1}==============" \
.format(self.case_number, self._testMethodName))
if not self.skip_buckets_handle and not self.skip_init_check_cbserver:
self.cluster_util._cluster_cleanup(self.bucket_util)
# avoid any cluster operations in setup for new upgrade
# & upgradeXDCR tests
if str(self.__class__).find('newupgradetests') != -1 or \
str(self.__class__).find('upgradeXDCR') != -1 or \
str(self.__class__).find('Upgrade_EpTests') != -1 or \
hasattr(self, 'skip_buckets_handle') and \
self.skip_buckets_handle:
log.info("any cluster operation in setup will be skipped")
self.primary_index_created = True
log.info("============== basetestcase setup was finished for test #{0} {1} ==============" \
.format(self.case_number, self._testMethodName))
return
# avoid clean up if the previous test has been tear down
if self.case_number == 1 or self.case_number > 1000:
if self.case_number > 1000:
log.warn("teardDown for previous test failed. will retry..")
self.case_number -= 1000
self.cleanup = True
if not self.skip_init_check_cbserver:
self.tearDownEverything()
self.task = ServerTasks()
if not self.skip_init_check_cbserver:
log.info("initializing cluster")
self.cluster_util.reset_cluster(self.targetMaster, self.reset_services)
master_services = self.cluster_util.get_services(self.servers[:1], \
self.services_init, \
start_node=0)
if master_services != None:
master_services = master_services[0].split(",")
self.quota = self._initialize_nodes(self.task, self.cluster.servers, \
self.disabled_consistent_view, \
self.rebalanceIndexWaitingDisabled, \
self.rebalanceIndexPausingDisabled, \
self.maxParallelIndexers, \
self.maxParallelReplicaIndexers, \
self.port, \
self.quota_percent, \
services=master_services)
self.cluster_util.change_env_variables()
self.cluster_util.change_checkpoint_params()
# Add built-in user
if not self.skip_init_check_cbserver:
self.add_built_in_server_user(node=self.cluster.master)
log.info("done initializing cluster")
else:
self.quota = ""
if self.input.param("log_info", None):
self.cluster_util.change_log_info()
if self.input.param("log_location", None):
self.cluster_util.change_log_location()
if self.input.param("stat_info", None):
self.cluster_util.change_stat_info()
if self.input.param("port_info", None):
self.cluster_util.change_port_info()
if self.input.param("port", None):
self.port = str(self.input.param("port", None))
try:
if (str(self.__class__).find('rebalanceout.RebalanceOutTests') != -1) or \
(str(self.__class__).find('memorysanitytests.MemorySanity') != -1) or \
str(self.__class__).find('negativetests.NegativeTests') != -1 or \
str(self.__class__).find('warmuptest.WarmUpTests') != -1 or \
str(self.__class__).find('failover.failovertests.FailoverTests') != -1 or \
str(self.__class__).find('observe.observeseqnotests.ObserveSeqNoTests') != -1 or \
str(self.__class__).find('epengine.lwwepengine.LWW_EP_Engine') != -1:
self.services = self.get_services(self.servers, self.services_init)
# rebalance all nodes into the cluster before each test
self.task.rebalance(self.servers[:self.num_servers], self.servers[1:self.num_servers], [],
services=self.services)
elif self.nodes_init > 1 and not self.skip_init_check_cbserver:
self.services = self.get_services(self.servers[:self.nodes_init], self.services_init)
self.task.rebalance(self.servers[:1], \
self.servers[1:self.nodes_init], \
[], services=self.services)
elif str(self.__class__).find('ViewQueryTests') != -1 and \
not self.input.param("skip_rebalance", False):
self.services = self.get_services(self.servers, self.services_init)
self.task.rebalance(self.servers, self.servers[1:],
[], services=self.services)
self.cluster_util.setDebugLevel(service_type="index")
except BaseException, e:
# increase case_number to retry tearDown in setup for the next test
self.case_number += 1000
self.fail(e)
log.info("============== basetestcase setup was finished for test #{0} {1} ==============" \
.format(self.case_number, self._testMethodName))
if not self.skip_init_check_cbserver:
self._log_start(self)
self.sleep(10)