def test_signal_impact_on_cbas(self):
self.log.info("Add nodes, create cbas bucket and dataset")
self.set_up_test()
self.log.info("Wait for ingestion to complete and verify count")
self.cbas_util.wait_for_ingestion_complete([self.dataset_name],
self.num_items)
self.assertTrue(
self.cbas_util.validate_cbas_dataset_items_count(
self.dataset_name, self.num_items))
self.log.info("Establish a remote connection")
con_cbas_node1 = RemoteMachineShellConnection(self.cbas_node)
con_cbas_node2 = RemoteMachineShellConnection(self.cbas_servers[0])
self.log.info("SIGSTOP ANALYTICS SERVICE")
con_cbas_node1.kill_process(self.process, self.service, 19)
con_cbas_node2.kill_process(self.process, self.service, 19)
self.log.info("Add more documents in the default bucket")
self.perform_doc_ops_in_all_cb_buckets(self.num_items,
"create",
self.num_items,
self.num_items * 2,
exp=0,
batch_size=self.batch_size)
self.log.info("SIGCONT ANALYTICS")
con_cbas_node1.kill_process(self.process, self.service, 18)
con_cbas_node2.kill_process(self.process, self.service, 18)
self.sleep(15)
self.log.info("Wait for ingestion to complete and verify count")
self.cbas_util.wait_for_ingestion_complete([self.dataset_name],
self.num_items * 2)
self.assertTrue(
self.cbas_util.validate_cbas_dataset_items_count(
self.dataset_name, self.num_items * 2))
self.log.info("SIGSTOP ANALYTICS SERVICE")
con_cbas_node1.kill_process(self.process, self.service, 19)
con_cbas_node2.kill_process(self.process, self.service, 19)
self.log.info("Delete documents in the default bucket")
self.perform_doc_ops_in_all_cb_buckets(self.num_items,
"delete",
0,
self.num_items,
exp=0,
batch_size=self.batch_size)
self.log.info("SIGCONT ANALYTICS")
con_cbas_node1.kill_process(self.process, self.service, 18)
con_cbas_node2.kill_process(self.process, self.service, 18)
self.sleep(15)
self.log.info("Wait for ingestion to complete and verify count")
self.cbas_util.wait_for_ingestion_complete([self.dataset_name],
self.num_items)
self.assertTrue(
self.cbas_util.validate_cbas_dataset_items_count(
self.dataset_name, self.num_items))
def test_analytics_recovery_on_busy_system(self):
self.log.info("Load data, create cbas buckets, and datasets")
self.set_up_test()
self.log.info("Wait for ingestion to complete")
self.cbas_util.wait_for_ingestion_complete([self.dataset_name],
self.num_items)
self.assertTrue(
self.cbas_util.validate_cbas_dataset_items_count(
self.dataset_name, self.num_items))
self.log.info("Get the nodes on which kill is to be run")
self.nodes_to_kill_service_on = []
if self.kill_on_cc:
neglect_failures = True
self.nodes_to_kill_service_on.append(self.cbas_node)
if self.kill_on_nc:
for cbas_server in self.cbas_servers:
self.nodes_to_kill_service_on.append(cbas_server)
self.log.info("Run concurrent queries to simulate busy system")
statement = "select sleep(count(*),50000) from {0} where mutated=0;".format(
self.dataset_name)
try:
self.cbas_util._run_concurrent_queries(statement,
"async",
10,
batch_size=10)
except Exception as e:
if neglect_failures:
self.log.info(
"Neglecting failed queries, to handle killing Java/Cbas process kill on CC & NC node %s"
% e)
else:
raise e
self.log.info("Establish a remote connection on node and kill service")
for node in self.nodes_to_kill_service_on:
shell = RemoteMachineShellConnection(node)
shell.kill_process(self.process, self.service, signum=self.signum)
self.sleep(
5,
"Sleeping for 5 seconds as after killing the service the service takes some time to exit and the service checks get pass by that time."
)
self.log.info(
"Wait for request to complete and cluster to be active: Using private ping() function"
)
service_up = False
start_time = time.time()
while time.time() < start_time + 120:
try:
status, metrics, _, cbas_result, _ = self.cbas_util.execute_statement_on_cbas_util(
"set `import-private-functions` `true`;ping();",
timeout=600,
analytics_timeout=600)
if status == "success":
service_up = True
break
except:
pass
self.sleep(1)
self.assertTrue(
service_up,
msg=
"CBAS service was not up even after 120 seconds of process kill. Failing the test possible a bug"
)
self.log.info("Observe no reingestion on node after restart")
items_in_cbas_bucket, _ = self.cbas_util.get_num_items_in_cbas_dataset(
self.dataset_name)
self.assertTrue(
items_in_cbas_bucket > 0,
msg=
"Items in CBAS bucket must greather than 0. If not re-ingestion has happened"
)
self.assertTrue(
self.cbas_util.validate_cbas_dataset_items_count(
self.dataset_name, self.num_items))
self.log.info("Add more documents in the default bucket")
self.perform_doc_ops_in_all_cb_buckets(self.num_items,
"create",
self.num_items,
self.num_items * 2,
exp=0,
batch_size=self.batch_size)
self.log.info("Wait for ingestion to complete")
self.assertTrue(
self.cbas_util.validate_cbas_dataset_items_count(
self.dataset_name, self.num_items * 2))