def test_restart_of_all_nodes(self):
self.log.info("Add nodes, create cbas bucket and dataset")
self.set_up_test()
self.log.info("Wait for ingestion to complete and verify count")
self.cbas_util.wait_for_ingestion_complete([self.dataset_name],
self.num_items)
self.assertTrue(
self.cbas_util.validate_cbas_dataset_items_count(
self.dataset_name, self.num_items))
self.log.info("Restart nodes")
restart_kv = self.input.param("restart_kv", True)
restart_cbas = self.input.param("restart_cbas", True)
self.restart_servers = []
if restart_kv:
for kv_server in self.kv_servers:
self.restart_servers.append(kv_server)
if restart_cbas:
self.restart_servers.append(self.cbas_node)
for cbas_server in self.cbas_servers:
self.restart_servers.append(cbas_server)
for restart_node in self.restart_servers:
NodeHelper.reboot_server_new(restart_node, self)
self.sleep(15, message="Wait for service to be up and accept request")
self.log.info("Add more documents in the default bucket")
self.perform_doc_ops_in_all_cb_buckets(self.num_items,
"create",
self.num_items,
self.num_items * 2,
exp=0,
batch_size=self.batch_size)
self.log.info("Wait for ingestion to complete and verify count")
self.cbas_util.wait_for_ingestion_complete([self.dataset_name],
self.num_items * 2)
self.assertTrue(
self.cbas_util.validate_cbas_dataset_items_count(
self.dataset_name, self.num_items * 2))
self.log.info("Delete documents in the default bucket")
self.perform_doc_ops_in_all_cb_buckets(self.num_items,
"delete",
0,
self.num_items,
exp=0,
batch_size=self.batch_size)
self.log.info("Wait for ingestion to complete and verify count")
self.cbas_util.wait_for_ingestion_complete([self.dataset_name],
self.num_items)
self.assertTrue(
self.cbas_util.validate_cbas_dataset_items_count(
self.dataset_name, self.num_items))
def test_restart_kv_server_impact_on_bucket(self):
self.log.info(
'Add documents, create CBAS buckets, dataset and validate count')
self.setup_for_test()
self.log.info('Restart couchbase')
NodeHelper.reboot_server_new(self.master, self)
self.log.info('Validate document count')
count_n1ql = self.rest.query_tool(
'select count(*) from %s' %
(self.cb_bucket_name))['results'][0]['$1']
self.cbas_util.validate_cbas_dataset_items_count(
self.cbas_dataset_name, count_n1ql)
def test_reboot_cbas(self):
'''
Description: This test will add the second cbas node then start rebalance and cancel rebalance
before rebalance completes.
Steps:
1. Add first cbas node.
2. Start rebalance, wait for rebalance complete.
3. Create bucket, datasets, connect bucket. Data ingestion should start.
4. Reboot CBAS node addd in Step 1.
5. After reboot cbas node should be able to serve queries, validate items count.
Author: Ritesh Agarwal
'''
self.load_sample_buckets(bucketName=self.cb_bucket_name, total_items=self.travel_sample_docs_count)
self.add_node(self.cbas_node, services=["kv","cbas"])
self.setup_cbas_bucket_dataset_connect(self.cb_bucket_name, self.travel_sample_docs_count)
NodeHelper.reboot_server_new(self.cbas_node, self)
self.assertTrue(self.cbas_util.validate_cbas_dataset_items_count(self.cbas_dataset_name, self.travel_sample_docs_count),"Data loss in CBAS.")
def test_logging_configurations_are_restored_post_service_restarts(self):
self.log.info("Add a cbas node")
result = self.add_node(self.cbas_servers[0],
services=["cbas"],
rebalance=True)
self.assertTrue(result, msg="Failed to add CBAS node")
self.log.info("Delete all loggers")
self.cbas_util.delete_all_loggers_on_cbas()
self.log.info("Set the logging level using the json object")
status, content, response = self.cbas_util.set_log_level_on_cbas(
CbasLogging.DEFAULT_LOGGER_CONFIG_DICT)
self.assertTrue(status,
msg="Response status incorrect for SET request")
self.log.info("Delete specific logger")
logger_name = self.input.param("logger_name_to_delete",
"com.couchbase.client.core.node")
status, content, response = self.cbas_util.delete_specific_cbas_log_level(
logger_name)
self.assertTrue(status, msg="Status mismatch for DELETE")
del CbasLogging.DEFAULT_LOGGER_CONFIG_DICT[logger_name]
self.log.info("Update specific logger")
logger_name = self.input.param("logger_name_to_update",
"org.apache.hyracks")
logger_level_to_update = self.input.param("logger_level_to_update",
"FATAL")
status, response, content = self.cbas_util.set_specific_log_level_on_cbas(
logger_name, logger_level_to_update)
self.assertTrue(status, msg="Status mismatch for SET")
CbasLogging.DEFAULT_LOGGER_CONFIG_DICT[
logger_name] = logger_level_to_update
self.log.info("Add a new logger")
logger_name = self.input.param("logger_name_to_add",
"org.apache.hyracks123")
logger_level_to_add = self.input.param("logger_level_to_add", "ALL")
status, response, content = self.cbas_util.set_specific_log_level_on_cbas(
logger_name, logger_level_to_add)
self.assertTrue(status, msg="Status mismatch for SET")
CbasLogging.DEFAULT_LOGGER_CONFIG_DICT[
logger_name] = logger_level_to_add
self.log.info("Verify logging configuration that we set on cbas Node")
for name, level in CbasLogging.DEFAULT_LOGGER_CONFIG_DICT.items():
status, content, response = self.cbas_util.get_specific_cbas_log_level(
name)
self.assertTrue(status,
msg="Response status incorrect for GET request")
self.assertEquals(content,
level,
msg="Logger configuration mismatch for logger " +
name)
self.sleep(
timeout=10,
message=
"Waiting for logger configuration to be copied across cbas nodes")
self.log.info("Verify logging configuration on other cbas node")
for name, level in CbasLogging.DEFAULT_LOGGER_CONFIG_DICT.items():
status, content, response = cbas_utils(
self.master,
self.cbas_servers[0]).get_specific_cbas_log_level(name)
self.assertTrue(status,
msg="Response status incorrect for GET request")
self.assertEquals(content,
level,
msg="Logger configuration mismatch for logger " +
name)
self.log.info("Read input params")
process_name = self.input.param('process_name', None)
service_name = self.input.param('service_name', None)
restart_couchbase = self.input.param('restart_couchbase', False)
reboot = self.input.param('reboot', False)
kill_services = self.input.param('kill_services', False)
self.log.info("Establish a remote connection")
shell_cc = RemoteMachineShellConnection(self.cbas_node)
shell_nc = RemoteMachineShellConnection(self.cbas_servers[0])
if kill_services:
self.log.info("Kill the %s service on CC cbas node" % service_name)
shell_cc.kill_process(process_name, service_name)
self.log.info("Kill the %s service on other cbas node" %
service_name)
shell_nc.kill_process(process_name, service_name)
if restart_couchbase:
self.log.info("Restart couchbase service")
status, _, _ = self.cbas_util.restart_analytics_cluster_uri()
self.assertTrue(status, msg="Failed to restart cbas")
if reboot:
self.log.info("Reboot couchbase CC node")
NodeHelper.reboot_server_new(self.cbas_node, self)
self.log.info("Reboot couchbase NC node")
NodeHelper.reboot_server_new(self.cbas_servers[0], self)
self.log.info(
"Wait for request to complete and cluster to be active: Using private ping() function"
)
cluster_recover_start_time = time.time()
while time.time() < cluster_recover_start_time + 180:
try:
status, metrics, _, cbas_result, _ = self.cbas_util.execute_statement_on_cbas_util(
"set `import-private-functions` `true`;ping()")
if status == "success":
break
except:
self.sleep(3, message="Wait for service to up")
self.log.info("Verify logging configuration post service kill")
for name, level in CbasLogging.DEFAULT_LOGGER_CONFIG_DICT.items():
status, content, response = self.cbas_util.get_specific_cbas_log_level(
name)
self.assertTrue(status,
msg="Response status incorrect for GET request")
self.assertEquals(content,
level,
msg="Logger configuration mismatch for logger " +
name)
self.sleep(
timeout=10,
message=
"Waiting for logger configuration to be copied across cbas nodes")
self.log.info(
"Verify logging configuration on other cbas node post service kill"
)
for name, level in CbasLogging.DEFAULT_LOGGER_CONFIG_DICT.items():
status, content, response = cbas_utils(
self.master,
self.cbas_servers[0]).get_specific_cbas_log_level(name)
self.assertTrue(status,
msg="Response status incorrect for GET request")
self.assertEquals(content,
level,
msg="Logger configuration mismatch for logger " +
name)