def upgrade_all_nodes_post_463(self):
servers_in = self.servers[1:]
self._install(self.servers)
self.cluster.rebalance(self.servers, servers_in, [])
for server in self.servers:
self.secretmgmt_base_obj.setup_pass_node(server, self.password)
self.secretmgmt_base_obj.restart_server_with_env(
self.master, self.password)
temp_result = self.secretmgmt_base_obj.check_log_files(
self.master, "/babysitter.log", "Booted")
self.assertTrue(
temp_result,
"Babysitter.log does not contain node initialization code")
upgrade_threads = self._async_update(
upgrade_version=self.upgrade_version, servers=self.servers)
for threads in upgrade_threads:
threads.join()
for server in self.servers:
rest = RestConnection(server)
temp = rest.cluster_status()
self.log.info("Initial status of {0} cluster is {1}".format(
server.ip, temp['nodes'][0]['status']))
while (temp['nodes'][0]['status'] == 'warmup'):
self.log.info("Waiting for cluster to become healthy")
self.sleep(5)
temp = rest.cluster_status()
self.log.info("current status of {0} is {1}".format(
server.ip, temp['nodes'][0]['status']))
def setUp(self):
super(XDCRTests, self).setUp()
self.bucket = Bucket()
self._initialize_nodes()
self.master = self.servers[0]
for server in self.servers:
rest=RestConnection(server)
cluster_status = rest.cluster_status()
self.log.info("Initial status of {0} cluster is {1}".format(server.ip,
cluster_status['nodes'][0]['status']))
while cluster_status['nodes'][0]['status'] == 'warmup':
self.log.info("Waiting for cluster to become healthy")
self.sleep(5)
cluster_status = rest.cluster_status()
self.log.info("current status of {0} is {1}".format(server.ip,
cluster_status['nodes'][0]['status']))
# Delete all buckets before creating new buckets
self.log.info("Deleting all existing buckets")
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
self.log.info("Creating new buckets")
src_bucket = self.input.param('src_bucket', self.bucket)
dest_bucket = self.input.param('dest_bucket', self.bucket)
if src_bucket:
RestConnection(self.servers[0]).create_bucket(bucket='default', ramQuotaMB=500)
if dest_bucket:
RestConnection(self.servers[1]).create_bucket(bucket='default', ramQuotaMB=500)
helper = BaseHelper(self)
helper.login()
def setUp(self):
super(XDCRTests, self).setUp()
self.bucket = Bucket()
self._initialize_nodes()
self.master = self.servers[0]
for server in self.servers:
rest = RestConnection(server)
cluster_status = rest.cluster_status()
self.log.info("Initial status of {0} cluster is {1}".format(
server.ip, cluster_status['nodes'][0]['status']))
while cluster_status['nodes'][0]['status'] == 'warmup':
self.log.info("Waiting for cluster to become healthy")
self.sleep(5)
cluster_status = rest.cluster_status()
self.log.info("current status of {0} is {1}".format(
server.ip, cluster_status['nodes'][0]['status']))
# Delete all buckets before creating new buckets
self.log.info("Deleting all existing buckets")
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
self.log.info("Creating new buckets")
src_bucket = self.input.param('src_bucket', self.bucket)
dest_bucket = self.input.param('dest_bucket', self.bucket)
if src_bucket:
RestConnection(self.servers[0]).create_bucket(bucket='default',
ramQuotaMB=500)
if dest_bucket:
RestConnection(self.servers[1]).create_bucket(bucket='default',
ramQuotaMB=500)
helper = BaseHelper(self)
helper.login()
def wait_for_warmup_or_assert(master, warmup_count, timeout, testcase):
time_start = time.time()
time_max_end = time_start + timeout
bucket_name = testcase.rest.get_buckets()[0].name
while time.time() < time_max_end:
num_nodes_with_warmup = 0
for node in testcase.rest.get_bucket(bucket_name).nodes:
if node.status == 'warmup':
num_nodes_with_warmup += 1
if num_nodes_with_warmup == warmup_count:
testcase.log.info("{0} nodes warmup as expected".format(
num_nodes_with_warmup))
testcase.log.info("expected warmup in {0} seconds, actual time {1} seconds".format \
(timeout - AutoReprovisionBaseTest.MAX_FAIL_DETECT_TIME,
time.time() - time_start))
return
time.sleep(2)
rest = RestConnection(master)
rest.print_UI_logs()
testcase.log.warn("pools/default from {0} : {1}".format(
master.ip, rest.cluster_status()))
testcase.fail("{0} nodes warmup, expected {1} in {2} seconds".format(
num_nodes_with_warmup, warmup_count,
time.time() - time_start))
def wait_for_failover_or_assert(master, autofailover_count, timeout,
testcase):
time_start = time.time()
time_max_end = time_start + timeout
failover_count = 0
while time.time() < time_max_end:
failover_count = AutoFailoverBaseTest.get_failover_count(master)
if failover_count == autofailover_count:
testcase.log.info(
"{0} nodes failed over as expected".format(failover_count))
testcase.log.info("expected failover in {0} seconds, actual time {1} seconds".format\
(timeout - AutoFailoverBaseTest.MAX_FAIL_DETECT_TIME, time.time() - time_start))
return
time.sleep(2)
rest = RestConnection(master)
testcase.log.info("Latest logs from UI:")
for i in rest.get_logs():
testcase.log.error(i)
testcase.log.warn("pools/default from {0} : {1}".format(
master.ip, rest.cluster_status()))
testcase.fail(
"{0} nodes failed over, expected {1} in {2} seconds".format(
failover_count, autofailover_count,
time.time() - time_start))
def get_indexer_mem_quota(self, indexer_node):
"""
Get Indexer memory Quota
:param indexer_node:
"""
rest = RestConnection(indexer_node)
content = rest.cluster_status()
return int(content['indexMemoryQuota'])
def get_failover_count(self):
rest = RestConnection(self.master)
cluster_status = rest.cluster_status()
failover_count = 0
# check for inactiveFailed
for node in cluster_status['nodes']:
if node['clusterMembership'] == "inactiveFailed":
failover_count += 1
return failover_count
def upgrade_half_nodes(self):
serv_upgrade = self.servers[2:4]
servers_in = self.servers[1:]
self._install(self.servers)
self.cluster.rebalance(self.servers, servers_in, [])
upgrade_threads = self._async_update(upgrade_version=self.upgrade_version, servers=serv_upgrade)
for threads in upgrade_threads:
threads.join()
for server in serv_upgrade:
rest = RestConnection(server)
temp = rest.cluster_status()
self.log.info("Initial status of {0} cluster is {1}".format(server.ip, temp['nodes'][0]['status']))
while (temp['nodes'][0]['status'] == 'warmup'):
self.log.info("Waiting for cluster to become healthy")
self.sleep(5)
temp = rest.cluster_status()
self.log.info("current status of {0} is {1}".format(server.ip, temp['nodes'][0]['status']))
def get_indexer_mem_quota(self):
"""
Sets Indexer memory Quota
:param memQuota:
:return:
int indexer memory quota
"""
rest = RestConnection(self.oomServer)
content = rest.cluster_status()
return int(content['indexMemoryQuota'])
def get_indexer_mem_quota(self):
"""
Sets Indexer memory Quota
:param memQuota:
:return:
int indexer memory quota
"""
rest = RestConnection(self.oomServer)
content = rest.cluster_status()
return int(content['indexMemoryQuota'])
def get_failover_count(master):
rest = RestConnection(master)
cluster_status = rest.cluster_status()
failover_count = 0
# check for inactiveFailed
for node in cluster_status['nodes']:
if node['clusterMembership'] == "inactiveFailed":
failover_count += 1
return failover_count
def get_failover_count(master):
rest = RestConnection(master)
cluster_status = rest.cluster_status()
failover_count = 0
# check for inactiveFailed
for node in cluster_status['nodes']:
log.info("'clusterMembership' for node {0} is {1}".format(node["otpNode"], node['status']))
if node['status'] == "unhealthy":
failover_count += 1
return failover_count
def get_failover_count(self, master):
rest = RestConnection(master)
cluster_status = rest.cluster_status()
failover_count = 0
# check for inactiveFailed
for node in cluster_status['nodes']:
self.log.info("'clusterMembership' for node {0} is {1}".format(node["otpNode"], node['clusterMembership']))
if node['clusterMembership'] == "inactiveFailed":
failover_count += 1
return failover_count
def get_failover_count(master):
rest = RestConnection(master)
cluster_status = rest.cluster_status()
log = logger.Logger.get_logger()
failover_count = 0
# check for inactiveFailed
for node in cluster_status["nodes"]:
log.info("'clusterMembership' for node {0} is {1}".format(node["otpNode"], node["clusterMembership"]))
if node["clusterMembership"] == "inactiveFailed":
failover_count += 1
return failover_count
def wait_for_failover_or_assert(master, autofailover_count, age, testcase):
testcase.log.info("waiting for {0} seconds for autofailover".format((age + 30)))
time.sleep(age + 30)
rest = RestConnection(master)
cluster_status = rest.cluster_status()
failover_count = 0
# check for inactiveFailed
for node in cluster_status['nodes']:
testcase.log.info("{0} is in state {1} and {2}".format(node['hostname'],node['status'],node['clusterMembership']))
if node['clusterMembership'] == "inactiveFailed":
failover_count += 1
testcase.assertTrue(failover_count == autofailover_count, "{0} nodes failed over, expected {1}".format(failover_count, autofailover_count))
def wait_for_failover_or_assert(self, master, autofailover_count, timeout):
time_start = time.time()
time_max_end = time_start + 300
failover_count = 0
while time.time() < time_max_end:
failover_count = self.get_failover_count(master)
if failover_count == autofailover_count:
break
self.sleep(30)
if failover_count != autofailover_count:
rest = RestConnection(master)
self.log.warn("pools/default from {0} : {1}".format(master.ip, rest.cluster_status()))
self.fail("{0} node(s) failed over, expected {1} in {2} seconds".
format(failover_count, autofailover_count, time.time() - time_start))
else:
self.log.info("{0} node(s) failed over as expected".format(failover_count))
def wait_for_failover_or_assert(self, master, autofailover_count, timeout):
time_start = time.time()
time_max_end = time_start + 300
failover_count = 0
while time.time() < time_max_end:
failover_count = self.get_failover_count(master)
if failover_count == autofailover_count:
break
self.sleep(30)
if failover_count != autofailover_count:
rest = RestConnection(master)
self.log.warn("pools/default from {0} : {1}".format(master.ip, rest.cluster_status()))
self.fail("{0} node(s) failed over, expected {1} in {2} seconds".
format(failover_count, autofailover_count, time.time() - time_start))
else:
self.log.info("{0} node(s) failed over as expected".format(failover_count))
def _get_current_auto_compaction_percentage(self):
""" check at bucket level and cluster level for compaction percentage """
auto_compact_percentage = None
rest = RestConnection(self.server)
content = rest.get_bucket_json(self.bucket)
if content["autoCompactionSettings"] != False:
auto_compact_percentage =\
content["autoCompactionSettings"]["viewFragmentationThreshold"]["percentage"]
else:
# try to read cluster level compaction settings
content = rest.cluster_status()
auto_compact_percentage =\
content["autoCompactionSettings"]["viewFragmentationThreshold"]["percentage"]
return auto_compact_percentage
def wait_for_failover_or_assert(master, autofailover_count, timeout, testcase):
time_start = time.time()
time_max_end = time_start + timeout
failover_count = 0
while time.time() < time_max_end:
failover_count = AutoFailoverBaseTest.get_failover_count(master)
if failover_count == autofailover_count:
testcase.log.info("{0} nodes failed over as expected".format(failover_count))
testcase.log.info("expected failover in {0} seconds, actual time {1} seconds".format\
(timeout - AutoFailoverBaseTest.MAX_FAIL_DETECT_TIME, time.time() - time_start))
return
time.sleep(2)
rest = RestConnection(master)
rest.print_UI_logs()
testcase.log.warn("pools/default from {0} : {1}".format(master.ip, rest.cluster_status()))
testcase.fail("{0} nodes failed over, expected {1} in {2} seconds".
format(failover_count, autofailover_count, time.time() - time_start))
def scenario_failover_node_delta_recovery(self, kwargs):
services = kwargs.get("services")
nodes_in_failed_state = list()
rest = RestConnection(self.cluster.master)
nodes_status = rest.cluster_status()["nodes"]
for node in nodes_status:
if node["clusterMembership"] == "inactiveFailed":
nodes_in_failed_state.append(node["otpNode"].split("@")[1])
self.log.info("Nodes in failover state: %s" % nodes_in_failed_state)
if len(services) > len(nodes_in_failed_state):
self.log.warning(
"Failover nodes '%s' < '%s' expected node to recover" %
(len(nodes_in_failed_state), len(services)))
for index, service in enumerate(services):
rest.set_recovery_type(
otpNode="ns_1@" + nodes_in_failed_state[index],
recoveryType=CbServer.Failover.RecoveryType.DELTA)
def wait_for_failover_or_assert(master, autofailover_count, timeout, testcase):
time_start = time.time()
time_max_end = time_start + timeout + 60
failover_count = 0
while time.time() < time_max_end:
failover_count = AutoFailoverBaseTest.get_failover_count(master)
if failover_count == autofailover_count:
break
time.sleep(2)
if failover_count != autofailover_count:
rest = RestConnection(master)
testcase.log.info("Latest logs from UI:")
for i in rest.get_logs(): testcase.log.error(i)
testcase.log.warn("pools/default from {0} : {1}".format(master.ip, rest.cluster_status()))
testcase.fail("{0} nodes failed over, expected {1} in {2} seconds".
format(failover_count, autofailover_count, time.time() - time_start))
else:
testcase.log.info("{O} nodes failed over as expected")
def wait_for_warmup_or_assert(master, warmup_count, timeout, testcase):
time_start = time.time()
time_max_end = time_start + timeout
bucket_name = testcase.rest.get_buckets()[0].name
while time.time() < time_max_end:
num_nodes_with_warmup = 0
for node in testcase.rest.get_bucket(bucket_name).nodes:
if node.status == 'warmup':
num_nodes_with_warmup += 1
if num_nodes_with_warmup == warmup_count:
testcase.log.info("{0} nodes warmup as expected".format(num_nodes_with_warmup))
testcase.log.info("expected warmup in {0} seconds, actual time {1} seconds".format \
(timeout - AutoReprovisionBaseTest.MAX_FAIL_DETECT_TIME,
time.time() - time_start))
return
time.sleep(2)
rest = RestConnection(master)
rest.print_UI_logs()
testcase.log.warn("pools/default from {0} : {1}".format(master.ip, rest.cluster_status()))
testcase.fail("{0} nodes warmup, expected {1} in {2} seconds".
format(num_nodes_with_warmup, warmup_count, time.time() - time_start))
def testSettingCompacttion(self):
'''setting-compacttion OPTIONS:
--compaction-db-percentage=PERCENTAGE at which point database compaction is triggered
--compaction-db-size=SIZE[MB] at which point database compaction is triggered
--compaction-view-percentage=PERCENTAGE at which point view compaction is triggered
--compaction-view-size=SIZE[MB] at which point view compaction is triggered
--compaction-period-from=HH:MM allow compaction time period from
--compaction-period-to=HH:MM allow compaction time period to
--enable-compaction-abort=[0|1] allow compaction abort when time expires
--enable-compaction-parallel=[0|1] allow parallel compaction for database and view'''
compaction_db_percentage = self.input.param("compaction-db-percentage", None)
compaction_db_size = self.input.param("compaction-db-size", None)
compaction_view_percentage = self.input.param("compaction-view-percentage", None)
compaction_view_size = self.input.param("compaction-view-size", None)
compaction_period_from = self.input.param("compaction-period-from", None)
compaction_period_to = self.input.param("compaction-period-to", None)
enable_compaction_abort = self.input.param("enable-compaction-abort", None)
enable_compaction_parallel = self.input.param("enable-compaction-parallel", None)
bucket = self.input.param("bucket", "default")
output = self.input.param("output", '')
rest = RestConnection(self.master)
remote_client = RemoteMachineShellConnection(self.master)
self.testBucketCreation()
cli_command = "setting-compacttion"
options = "--bucket={0}".format(bucket)
options += (" --compaction-db-percentage={0}".format(compaction_db_percentage), "")[compaction_db_percentage is None]
options += (" --compaction-db-size={0}".format(compaction_db_size), "")[compaction_db_size is None]
options += (" --compaction-view-percentage={0}".format(compaction_view_percentage), "")[compaction_view_percentage is None]
options += (" --compaction-view-size={0}".format(compaction_view_size), "")[compaction_view_size is None]
options += (" --compaction-period-from={0}".format(compaction_period_from), "")[compaction_period_from is None]
options += (" --compaction-period-to={0}".format(compaction_period_to), "")[compaction_period_to is None]
options += (" --enable-compaction-abort={0}".format(enable_compaction_abort), "")[enable_compaction_abort is None]
options += (" --enable-compaction-parallel={0}".format(enable_compaction_parallel), "")[enable_compaction_parallel is None]
output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, options=options, cluster_host="localhost", user="******", password="******")
self.assertEqual(output, ['SUCCESS: bucket-edit'])
cluster_status = rest.cluster_status()
remote_client.disconnect()
def offline(self, node_to_upgrade, version, rebalance_required=False):
rest = RestConnection(node_to_upgrade)
shell = RemoteMachineShellConnection(node_to_upgrade)
appropriate_build = self.__get_build(version, shell)
self.assertTrue(appropriate_build.url,
msg="Unable to find build %s" % version)
self.assertTrue(shell.download_build(appropriate_build),
"Failed while downloading the build!")
self.log.info("Starting node upgrade")
upgrade_success = shell.couchbase_upgrade(
appropriate_build, save_upgrade_config=False,
forcefully=self.is_downgrade)
shell.disconnect()
if not upgrade_success:
self.log_failure("Upgrade failed")
return
self.log.info("Wait for ns_server to accept connections")
if not rest.is_ns_server_running(timeout_in_seconds=120):
self.log_failure("Server not started post upgrade")
return
self.log.info("Validate the cluster rebalance status")
if not rest.cluster_status()["balanced"]:
if rebalance_required:
otp_nodes = [node.id for node in rest.node_statuses()]
rest.rebalance(otpNodes=otp_nodes, ejectedNodes=[])
rebalance_passed = rest.monitorRebalance()
if not rebalance_passed:
self.log_failure(
"Rebalance failed post node upgrade of {0}"
.format(node_to_upgrade))
return
else:
self.log_failure("Cluster reported (/pools/default) balanced=false")
return
def setUp(self):
super(QueryTests, self).setUp()
self.expiry = self.input.param("expiry", 0)
self.batch_size = self.input.param("batch_size", 1)
self.scan_consistency = self.input.param("scan_consistency",
"request_plus")
self.skip_cleanup = self.input.param("skip_cleanup", False)
self.run_async = self.input.param("run_async", True)
self.version = self.input.param("cbq_version", "git_repo")
for server in self.servers:
rest = RestConnection(server)
temp = rest.cluster_status()
self.log.info("Initial status of {0} cluster is {1}".format(
server.ip, temp['nodes'][0]['status']))
while (temp['nodes'][0]['status'] == 'warmup'):
self.log.info("Waiting for cluster to become healthy")
self.sleep(5)
temp = rest.cluster_status()
self.log.info("current status of {0} is {1}".format(
server.ip, temp['nodes'][0]['status']))
indexer_node = self.get_nodes_from_services_map(service_type="index",
get_all_nodes=True)
# Set indexer storage mode
indexer_rest = RestConnection(indexer_node[0])
doc = {"indexer.settings.storage_mode": self.gsi_type}
indexer_rest.set_index_settings_internal(doc)
doc = {"indexer.api.enableTestServer": True}
indexer_rest.set_index_settings_internal(doc)
self.indexer_scanTimeout = self.input.param("indexer_scanTimeout",
None)
if self.indexer_scanTimeout is not None:
for server in indexer_node:
rest = RestConnection(server)
rest.set_index_settings({
"indexer.settings.scan_timeout":
self.indexer_scanTimeout
})
if self.input.tuq_client and "client" in self.input.tuq_client:
self.shell = RemoteMachineShellConnection(
self.input.tuq_client["client"])
else:
self.shell = RemoteMachineShellConnection(self.master)
self.use_gsi_for_primary = self.input.param("use_gsi_for_primary",
True)
self.use_gsi_for_secondary = self.input.param("use_gsi_for_secondary",
True)
self.create_primary_index = self.input.param("create_primary_index",
True)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.item_flag = self.input.param("item_flag", 4042322160)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.dataset = self.input.param("dataset", "default")
self.value_size = self.input.param("value_size", 1024)
self.doc_ops = self.input.param("doc_ops", False)
self.create_ops_per = self.input.param("create_ops_per", 0)
self.expiry_ops_per = self.input.param("expiry_ops_per", 0)
self.delete_ops_per = self.input.param("delete_ops_per", 0)
self.update_ops_per = self.input.param("update_ops_per", 0)
self.gens_load = self.generate_docs(self.docs_per_day)
if self.input.param("gomaxprocs", None):
self.n1ql_helper.configure_gomaxprocs()
self.full_docs_list = self.generate_full_docs_list(self.gens_load)
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
verify_data = False
if self.scan_consistency != "request_plus":
verify_data = True
self.load(self.gens_load,
flag=self.item_flag,
verify_data=verify_data,
batch_size=self.batch_size)
if self.doc_ops:
self.ops_dist_map = self.calculate_data_change_distribution(
create_per=self.create_ops_per,
update_per=self.update_ops_per,
delete_per=self.delete_ops_per,
expiry_per=self.expiry_ops_per,
start=0,
end=self.docs_per_day)
self.log.info(self.ops_dist_map)
self.docs_gen_map = self.generate_ops_docs(self.docs_per_day, 0)
self.full_docs_list_after_ops = self.generate_full_docs_list_after_ops(
self.docs_gen_map)
# Define Helper Method which will be used for running n1ql queries, create index, drop index
self.n1ql_helper = N1QLHelper(version=self.version,
shell=self.shell,
use_rest=self.use_rest,
max_verify=self.max_verify,
buckets=self.buckets,
item_flag=self.item_flag,
n1ql_port=self.n1ql_port,
full_docs_list=self.full_docs_list,
log=self.log,
input=self.input,
master=self.master)
self.n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
self.log.info(self.n1ql_node)
#self.n1ql_helper._start_command_line_query(self.n1ql_node)
# sleep to avoid race condition during bootstrap
if self.create_primary_index:
try:
self.n1ql_helper.create_primary_index(
using_gsi=self.use_gsi_for_primary, server=self.n1ql_node)
except Exception as ex:
self.log.info(ex)
raise ex
