def setUp(self):
"""
Since BaseTestCase will initialize at least one cluster, we pass service
for the master node of that cluster
"""
if not hasattr(self, "input"):
self.input = TestInputSingleton.input
"""
Cluster node services. Parameter value format
serv1:serv2-serv1:ser2|serv1:serv2-ser1:serv2
| -> separates services per cluster.
- -> separates services on each node of the cluster.
: -> separates services on a node.
"""
temp_service_init = [x.split("-") for x in self.input.param(
"services_init", "kv:n1ql:index").split("|")]
self.input.test_params.update(
{"services_init": temp_service_init[0][0]})
"""
Number of nodes per cluster. Parameter value format
num_nodes_cluster1|num_nodes_cluster2|....
| -> separates number of nodes per cluster.
"""
if not isinstance(self.input.param("nodes_init", 1), int):
temp_nodes_init = [int(x) for x in self.input.param(
"nodes_init", 1).split("|")]
else:
temp_nodes_init = [self.input.param("nodes_init", 1)]
super(CBASBaseTest, self).setUp()
if self._testMethodDoc:
self.log.info("Starting Test: %s - %s"
% (self._testMethodName, self._testMethodDoc))
else:
self.log.info("Starting Test: %s" % self._testMethodName)
self.services_init = temp_service_init
self.nodes_init = temp_nodes_init
"""
Parameterized Support for multiple cluster instead of creating
multiple clusters from ini file.
"""
self.num_of_clusters = self.input.param('num_of_clusters', 1)
"""
Since BaseTestCase will initialize at least one cluster, we need to
modify the initialized cluster server property to correctly reflect the
servers in that cluster.
"""
start = 0
end = self.nodes_init[0]
cluster = self.cb_clusters[self.cb_clusters.keys()[0]]
cluster.servers = self.servers[start:end]
cluster.nodes_in_cluster.append(cluster.master)
cluster.kv_nodes.append(cluster.master)
if "cbas" in cluster.master.services:
cluster.cbas_nodes.append(cluster.master)
"""
Since BaseTestCase will initialize at least one cluster, we need to
initialize only total clusters required - 1.
"""
cluster_name_format = "C%s"
for i in range(1, self.num_of_clusters):
start = end
end += self.nodes_init[i]
cluster_name = cluster_name_format % str(i+1)
cluster = CBCluster(
name=cluster_name,
servers=self.servers[start:end])
self.cb_clusters[cluster_name] = cluster
cluster.nodes_in_cluster.append(cluster.master)
cluster.kv_nodes.append(cluster.master)
if "cbas" in cluster.master.services:
cluster.cbas_nodes.append(cluster.master)
self.initialize_cluster(cluster_name, cluster,
services=self.services_init[i][0])
self.modify_cluster_settings(cluster)
self.available_servers = self.servers[end:]
"""
KV infra to be created per cluster.
Accepted values are -
bkt_spec : will create KV infra based on bucket spec. bucket_spec param needs to be passed.
default : will create a bucket named default on the cluster.
None : no buckets will be created on cluster
| -> separates number of nodes per cluster.
"""
if self.input.param("cluster_kv_infra", None):
self.cluster_kv_infra = self.input.param("cluster_kv_infra", None).split("|")
if len(self.cluster_kv_infra) < self.num_of_clusters:
self.cluster_kv_infra.extend(
[None] * (self.num_of_clusters - len(self.cluster_kv_infra)))
else:
self.cluster_kv_infra = [None] * self.num_of_clusters
# Common properties
self.num_concurrent_queries = self.input.param('num_queries', 5000)
self.concurrent_batch_size = self.input.param('concurrent_batch_size',
100)
self.index_fields = self.input.param('index_fields', None)
if self.index_fields:
self.index_fields = self.index_fields.split("-")
self.retry_time = self.input.param("retry_time", 300)
self.num_retries = self.input.param("num_retries", 1)
self.cbas_spec_name = self.input.param("cbas_spec", None)
self.expected_error = self.input.param("error", None)
self.bucket_spec = self.input.param("bucket_spec", None)
self.doc_spec_name = self.input.param("doc_spec_name", "initial_load")
self.set_default_cbas_memory = self.input.param(
'set_default_cbas_memory', False)
self.cbas_memory_quota_percent = int(self.input.param(
"cbas_memory_quota_percent", 100))
self.bucket_size = self.input.param("bucket_size", 250)
self.cbas_util = CbasUtil(self.task)
# Add nodes to the cluster as per node_init param.
for i, (cluster_name, cluster) in enumerate(self.cb_clusters.items()):
cluster.rest = RestConnection(cluster.master)
for j, server in enumerate(cluster.servers):
if server.ip != cluster.master.ip:
server.services = self.services_init[i][j].replace(":", ",")
if "cbas" in server.services:
cluster.cbas_nodes.append(server)
if "kv" in server.services:
cluster.kv_nodes.append(server)
rest = RestConnection(server)
rest.set_data_path(
data_path=server.data_path,
index_path=server.index_path,
cbas_path=server.cbas_path)
if self.set_default_cbas_memory:
self.log.info(
"Setting the min possible memory quota so that adding "
"more nodes to the cluster wouldn't be a problem.")
cluster.rest.set_service_mem_quota(
{
CbServer.Settings.KV_MEM_QUOTA: MIN_KV_QUOTA,
CbServer.Settings.FTS_MEM_QUOTA: FTS_QUOTA,
CbServer.Settings.INDEX_MEM_QUOTA: INDEX_QUOTA
})
self.log.info("Setting %d memory quota for CBAS" % CBAS_QUOTA)
cluster.cbas_memory_quota = CBAS_QUOTA
cluster.rest.set_service_mem_quota(
{
CbServer.Settings.CBAS_MEM_QUOTA: CBAS_QUOTA
})
else:
self.set_memory_for_services(
cluster, server, server.services)
if cluster.servers[1:]:
self.task.rebalance(
[cluster.master], cluster.servers[1:], [],
services=[server.services for server in cluster.servers[1:]])
cluster.nodes_in_cluster.extend(cluster.servers[1:])
if cluster.cbas_nodes:
cbas_cc_node_ip = None
retry = 0
while True and retry < 60:
cbas_cc_node_ip = self.cbas_util.retrieve_cc_ip_from_master(
cluster)
if cbas_cc_node_ip:
break
else:
self.sleep(10, "Waiting for CBAS service to come up")
retry += 1
for server in cluster.cbas_nodes:
if server.ip == cbas_cc_node_ip:
cluster.cbas_cc_node = server
break
if "cbas" in cluster.master.services:
self.cbas_util.cleanup_cbas(cluster)
cluster.otpNodes = cluster.rest.node_statuses()
if self.cluster_kv_infra[i] == "bkt_spec":
if self.bucket_spec is not None:
try:
self.collectionSetUp(cluster)
except Java_base_exception as exception:
self.handle_setup_exception(exception)
except Exception as exception:
self.handle_setup_exception(exception)
else:
self.fail("Error : bucket_spec param needed")
elif self.cluster_kv_infra[i] == "default":
self.bucket_util.create_default_bucket(
cluster,
bucket_type=self.bucket_type,
ram_quota=self.bucket_size,
replica=self.num_replicas,
conflict_resolution=self.bucket_conflict_resolution_type,
replica_index=self.bucket_replica_index,
storage=self.bucket_storage,
eviction_policy=self.bucket_eviction_policy,
flush_enabled=self.flush_enabled)
self.bucket_util.add_rbac_user(cluster.master)
# Wait for analytics service to be up.
if hasattr(cluster, "cbas_util"):
if not self.cbas_util.is_analytics_running(cluster):
self.fail("Analytics service did not come up even after 10\
mins of wait after initialisation")
self.log.info("=== CBAS_BASE setup was finished for test #{0} {1} ==="
.format(self.case_number, self._testMethodName))
def setUp(self):
"""
Since BaseTestCase will initialize at least one cluster, we pass service
for the master node of that cluster
"""
if not hasattr(self, "input"):
self.input = TestInputSingleton.input
""" In case of multi cluster setup, if cluster address family needs to be
set, then this parameter is required """
if self.input.param("cluster_ip_family", ""):
cluster_ip_family = self.input.param("cluster_ip_family",
"").split("|")
if cluster_ip_family[0] == "ipv4_only":
self.input.test_params.update({
"ipv4_only": True,
"ipv6_only": False
})
elif cluster_ip_family[0] == "ipv6_only":
self.input.test_params.update({
"ipv4_only": False,
"ipv6_only": True
})
elif cluster_ip_family[0] == "ipv4_ipv6":
self.input.test_params.update({
"ipv4_only": True,
"ipv6_only": True
})
else:
self.input.test_params.update({
"ipv4_only": False,
"ipv6_only": False
})
super(CBASBaseTest, self).setUp()
"""
Cluster node services. Parameter value format
serv1:serv2-serv1:ser2|serv1:serv2-ser1:serv2
| -> separates services per cluster.
- -> separates services on each node of the cluster.
: -> separates services on a node.
"""
self.services_init = [
x.split("-") for x in self.input.param("services_init",
"kv:n1ql:index").split("|")
]
"""
Number of nodes per cluster. Parameter value format
num_nodes_cluster1|num_nodes_cluster2|....
| -> separates number of nodes per cluster.
"""
if not isinstance(self.input.param("nodes_init", 1), int):
self.nodes_init = [
int(x) for x in self.input.param("nodes_init", 1).split("|")
]
else:
self.nodes_init = [self.input.param("nodes_init", 1)]
if self._testMethodDoc:
self.log.info("Starting Test: %s - %s" %
(self._testMethodName, self._testMethodDoc))
else:
self.log.info("Starting Test: %s" % self._testMethodName)
"""
Parameterized Support for multiple cluster instead of creating
multiple clusters from ini file.
"""
self.num_of_clusters = self.input.param('num_of_clusters', 1)
"""
Since BaseTestCase will initialize at least one cluster, we need to
modify the initialized cluster server property to correctly reflect the
servers in that cluster.
"""
start = 0
end = self.nodes_init[0]
cluster = self.cb_clusters[self.cb_clusters.keys()[0]]
cluster.servers = self.servers[start:end]
if "cbas" in cluster.master.services:
cluster.cbas_nodes.append(cluster.master)
"""
Since BaseTestCase will initialize at least one cluster, we need to
initialize only total clusters required - 1.
"""
cluster_name_format = "C%s"
for i in range(1, self.num_of_clusters):
start = end
end += self.nodes_init[i]
cluster_name = cluster_name_format % str(i + 1)
cluster = CBCluster(name=cluster_name,
servers=self.servers[start:end])
self.cb_clusters[cluster_name] = cluster
cluster.nodes_in_cluster.append(cluster.master)
cluster.kv_nodes.append(cluster.master)
self.initialize_cluster(cluster_name,
cluster,
services=self.services_init[i][0])
cluster.master.services = self.services_init[i][0].replace(
":", ",")
if "cbas" in cluster.master.services:
cluster.cbas_nodes.append(cluster.master)
if self.input.param("cluster_ip_family", ""):
# Enforce IPv4 or IPv6 or both
if cluster_ip_family[i] == "ipv4_only":
status, msg = self.cluster_util.enable_disable_ip_address_family_type(
cluster, True, True, False)
if cluster_ip_family[i] == "ipv6_only":
status, msg = self.cluster_util.enable_disable_ip_address_family_type(
cluster, True, False, True)
if cluster_ip_family[i] == "ipv4_ipv6":
status, msg = self.cluster_util.enable_disable_ip_address_family_type(
cluster, True, True, True)
if not status:
self.fail(msg)
self.modify_cluster_settings(cluster)
self.available_servers = self.servers[end:]
"""
KV infra to be created per cluster.
Accepted values are -
bkt_spec : will create KV infra based on bucket spec. bucket_spec param needs to be passed.
default : will create a bucket named default on the cluster.
None : no buckets will be created on cluster
| -> separates number of nodes per cluster.
"""
if self.input.param("cluster_kv_infra", None):
self.cluster_kv_infra = self.input.param("cluster_kv_infra",
None).split("|")
if len(self.cluster_kv_infra) < self.num_of_clusters:
self.cluster_kv_infra.extend(
[None] *
(self.num_of_clusters - len(self.cluster_kv_infra)))
else:
self.cluster_kv_infra = [None] * self.num_of_clusters
# Common properties
self.num_concurrent_queries = self.input.param('num_queries', 5000)
self.concurrent_batch_size = self.input.param('concurrent_batch_size',
100)
self.index_fields = self.input.param('index_fields', None)
if self.index_fields:
self.index_fields = self.index_fields.split("-")
self.retry_time = self.input.param("retry_time", 300)
self.num_retries = self.input.param("num_retries", 1)
self.cbas_spec_name = self.input.param("cbas_spec", None)
self.expected_error = self.input.param("error", None)
self.bucket_spec = self.input.param("bucket_spec", "analytics.default")
self.doc_spec_name = self.input.param("doc_spec_name", "initial_load")
self.set_default_cbas_memory = self.input.param(
'set_default_cbas_memory', False)
self.cbas_memory_quota_percent = int(
self.input.param("cbas_memory_quota_percent", 100))
self.bucket_size = self.input.param("bucket_size", 250)
self.cbas_util = CbasUtil(self.task)
self.service_mem_dict = {
"kv": [
CbServer.Settings.KV_MEM_QUOTA,
CbServer.Settings.MinRAMQuota.KV, 0
],
"fts": [
CbServer.Settings.FTS_MEM_QUOTA,
CbServer.Settings.MinRAMQuota.FTS, 0
],
"index": [
CbServer.Settings.INDEX_MEM_QUOTA,
CbServer.Settings.MinRAMQuota.INDEX, 0
],
"cbas": [
CbServer.Settings.CBAS_MEM_QUOTA,
CbServer.Settings.MinRAMQuota.CBAS, 0
],
}
# Add nodes to the cluster as per node_init param.
for i, (cluster_name, cluster) in enumerate(self.cb_clusters.items()):
cluster.rest = RestConnection(cluster.master)
cluster_services = self.cluster_util.get_services_map(cluster)
cluster_info = cluster.rest.get_nodes_self()
for service in cluster_services:
if service != "n1ql":
property_name = self.service_mem_dict[service][0]
service_mem_in_cluster = cluster_info.__getattribute__(
property_name)
self.service_mem_dict[service][2] = service_mem_in_cluster
j = 1
for server in cluster.servers:
if server.ip != cluster.master.ip:
server.services = self.services_init[i][j].replace(
":", ",")
j += 1
if "cbas" in server.services:
cluster.cbas_nodes.append(server)
if "kv" in server.services:
cluster.kv_nodes.append(server)
rest = RestConnection(server)
rest.set_data_path(data_path=server.data_path,
index_path=server.index_path,
cbas_path=server.cbas_path)
if self.set_default_cbas_memory:
self.log.info(
"Setting the min possible memory quota so that adding "
"more nodes to the cluster wouldn't be a problem.")
cluster.rest.set_service_mem_quota({
CbServer.Settings.KV_MEM_QUOTA:
CbServer.Settings.MinRAMQuota.KV,
CbServer.Settings.FTS_MEM_QUOTA:
CbServer.Settings.MinRAMQuota.FTS,
CbServer.Settings.INDEX_MEM_QUOTA:
CbServer.Settings.MinRAMQuota.INDEX
})
self.log.info("Setting %d memory quota for CBAS" %
CbServer.Settings.MinRAMQuota.CBAS)
cluster.cbas_memory_quota = CbServer.Settings.MinRAMQuota.CBAS
cluster.rest.set_service_mem_quota({
CbServer.Settings.CBAS_MEM_QUOTA:
CbServer.Settings.MinRAMQuota.CBAS
})
else:
self.set_memory_for_services(cluster, server,
server.services)
if cluster.servers[1:]:
self.task.rebalance(cluster,
cluster.servers[1:], [],
services=[
server.services
for server in cluster.servers[1:]
])
if cluster.cbas_nodes:
cbas_cc_node_ip = None
retry = 0
while True and retry < 60:
cbas_cc_node_ip = self.cbas_util.retrieve_cc_ip_from_master(
cluster)
if cbas_cc_node_ip:
break
else:
self.sleep(10, "Waiting for CBAS service to come up")
retry += 1
if not cbas_cc_node_ip:
self.fail("CBAS service did not come up even after 10 "
"mins.")
for server in cluster.cbas_nodes:
if server.ip == cbas_cc_node_ip:
cluster.cbas_cc_node = server
break
if "cbas" in cluster.master.services:
self.cbas_util.cleanup_cbas(cluster)
cluster.otpNodes = cluster.rest.node_statuses()
# Wait for analytics service to be up.
if hasattr(cluster, "cbas_cc_node"):
if not self.cbas_util.is_analytics_running(cluster):
self.fail("Analytics service did not come up even after 10\
mins of wait after initialisation")
if self.input.param("n2n_encryption", False):
self.security_util = SecurityUtils(self.log)
rest = RestConnection(cluster.master)
self.log.info("Disabling Auto-Failover")
if not rest.update_autofailover_settings(False, 120):
self.fail("Disabling Auto-Failover failed")
self.log.info("Setting node to node encryption level to all")
self.security_util.set_n2n_encryption_level_on_nodes(
cluster.nodes_in_cluster,
level=self.input.param("n2n_encryption_level", "control"))
CbServer.use_https = True
self.log.info("Enabling Auto-Failover")
if not rest.update_autofailover_settings(True, 300):
self.fail("Enabling Auto-Failover failed")
if self.input.param("analytics_loggers", None):
"""
This flag is used for setting analytics internal log levels.
These logs are helpful while dubugging issues as they
provide a deeper insight into working on CBAS service.
This flag can be used to set one or more logger for analytics.
logger_name_1:level-logger_name_2:level-......
"""
cbas_loggers = self.input.param("analytics_loggers",
None).split("-")
log_level_dict = dict()
for logger in cbas_loggers:
tmp = logger.split(":")
log_level_dict[tmp[0]] = tmp[1]
self.log.info("Setting following log levels for analytics - "
"{0}".format(log_level_dict))
status, content, response = self.cbas_util.set_log_level_on_cbas(
self.cluster, log_level_dict, timeout=120)
if not status:
self.fail("Error while setting log level for CBAS - "
"{0}".format(content))
self.log.info("Verifying whether log levels set successfully")
status, content, response = self.cbas_util.get_log_level_on_cbas(
self.cluster)
match_counter = 0
if status:
actual_log_levels = content["loggers"]
for logger in actual_log_levels:
if (logger["name"] in log_level_dict) and \
logger["level"] == log_level_dict[logger["name"]]:
match_counter += 1
if match_counter == len(log_level_dict):
self.log.info("All log levels were set successfully")
else:
self.fail("Some log levels were not set")
else:
self.fail("Error while fetching log levels")
self.disk_optimized_thread_settings = self.input.param(
"disk_optimized_thread_settings", False)
if self.disk_optimized_thread_settings:
self.set_num_writer_and_reader_threads(
cluster,
num_writer_threads="disk_io_optimized",
num_reader_threads="disk_io_optimized")
if self.cluster_kv_infra[i] == "bkt_spec":
if self.bucket_spec is not None:
try:
self.collectionSetUp(cluster)
except Java_base_exception as exception:
self.handle_setup_exception(exception)
except Exception as exception:
self.handle_setup_exception(exception)
else:
self.fail("Error : bucket_spec param needed")
elif self.cluster_kv_infra[i] == "default":
self.bucket_util.create_default_bucket(
cluster,
bucket_type=self.bucket_type,
ram_quota=self.bucket_size,
replica=self.num_replicas,
conflict_resolution=self.bucket_conflict_resolution_type,
replica_index=self.bucket_replica_index,
storage=self.bucket_storage,
eviction_policy=self.bucket_eviction_policy,
flush_enabled=self.flush_enabled)
self.bucket_util.add_rbac_user(cluster.master)
self.log.info(
"=== CBAS_BASE setup was finished for test #{0} {1} ===".format(
self.case_number, self._testMethodName))