def test_signal_impact_on_cbas(self): self.log.info("Add nodes, create cbas bucket and dataset") self.set_up_test() self.log.info("Wait for ingestion to complete and verify count") self.cbas_util.wait_for_ingestion_complete([self.dataset_name], self.num_items) self.assertTrue( self.cbas_util.validate_cbas_dataset_items_count( self.dataset_name, self.num_items)) self.log.info("Establish a remote connection") con_cbas_node1 = RemoteMachineShellConnection(self.cbas_node) con_cbas_node2 = RemoteMachineShellConnection(self.cbas_servers[0]) self.log.info("SIGSTOP ANALYTICS SERVICE") con_cbas_node1.kill_process(self.process, self.service, 19) con_cbas_node2.kill_process(self.process, self.service, 19) self.log.info("Add more documents in the default bucket") self.perform_doc_ops_in_all_cb_buckets(self.num_items, "create", self.num_items, self.num_items * 2, exp=0, batch_size=self.batch_size) self.log.info("SIGCONT ANALYTICS") con_cbas_node1.kill_process(self.process, self.service, 18) con_cbas_node2.kill_process(self.process, self.service, 18) self.sleep(15) self.log.info("Wait for ingestion to complete and verify count") self.cbas_util.wait_for_ingestion_complete([self.dataset_name], self.num_items * 2) self.assertTrue( self.cbas_util.validate_cbas_dataset_items_count( self.dataset_name, self.num_items * 2)) self.log.info("SIGSTOP ANALYTICS SERVICE") con_cbas_node1.kill_process(self.process, self.service, 19) con_cbas_node2.kill_process(self.process, self.service, 19) self.log.info("Delete documents in the default bucket") self.perform_doc_ops_in_all_cb_buckets(self.num_items, "delete", 0, self.num_items, exp=0, batch_size=self.batch_size) self.log.info("SIGCONT ANALYTICS") con_cbas_node1.kill_process(self.process, self.service, 18) con_cbas_node2.kill_process(self.process, self.service, 18) self.sleep(15) self.log.info("Wait for ingestion to complete and verify count") self.cbas_util.wait_for_ingestion_complete([self.dataset_name], self.num_items) self.assertTrue( self.cbas_util.validate_cbas_dataset_items_count( self.dataset_name, self.num_items))
def test_analytics_recovery_on_busy_system(self): self.log.info("Load data, create cbas buckets, and datasets") self.set_up_test() self.log.info("Wait for ingestion to complete") self.cbas_util.wait_for_ingestion_complete([self.dataset_name], self.num_items) self.assertTrue( self.cbas_util.validate_cbas_dataset_items_count( self.dataset_name, self.num_items)) self.log.info("Get the nodes on which kill is to be run") self.nodes_to_kill_service_on = [] if self.kill_on_cc: neglect_failures = True self.nodes_to_kill_service_on.append(self.cbas_node) if self.kill_on_nc: for cbas_server in self.cbas_servers: self.nodes_to_kill_service_on.append(cbas_server) self.log.info("Run concurrent queries to simulate busy system") statement = "select sleep(count(*),50000) from {0} where mutated=0;".format( self.dataset_name) try: self.cbas_util._run_concurrent_queries(statement, "async", 10, batch_size=10) except Exception as e: if neglect_failures: self.log.info( "Neglecting failed queries, to handle killing Java/Cbas process kill on CC & NC node %s" % e) else: raise e self.log.info("Establish a remote connection on node and kill service") for node in self.nodes_to_kill_service_on: shell = RemoteMachineShellConnection(node) shell.kill_process(self.process, self.service, signum=self.signum) self.sleep( 5, "Sleeping for 5 seconds as after killing the service the service takes some time to exit and the service checks get pass by that time." ) self.log.info( "Wait for request to complete and cluster to be active: Using private ping() function" ) service_up = False start_time = time.time() while time.time() < start_time + 120: try: status, metrics, _, cbas_result, _ = self.cbas_util.execute_statement_on_cbas_util( "set `import-private-functions` `true`;ping();", timeout=600, analytics_timeout=600) if status == "success": service_up = True break except: pass self.sleep(1) self.assertTrue( service_up, msg= "CBAS service was not up even after 120 seconds of process kill. Failing the test possible a bug" ) self.log.info("Observe no reingestion on node after restart") items_in_cbas_bucket, _ = self.cbas_util.get_num_items_in_cbas_dataset( self.dataset_name) self.assertTrue( items_in_cbas_bucket > 0, msg= "Items in CBAS bucket must greather than 0. If not re-ingestion has happened" ) self.assertTrue( self.cbas_util.validate_cbas_dataset_items_count( self.dataset_name, self.num_items)) self.log.info("Add more documents in the default bucket") self.perform_doc_ops_in_all_cb_buckets(self.num_items, "create", self.num_items, self.num_items * 2, exp=0, batch_size=self.batch_size) self.log.info("Wait for ingestion to complete") self.assertTrue( self.cbas_util.validate_cbas_dataset_items_count( self.dataset_name, self.num_items * 2))