def test_reboot_nodes(self): #Test for reboot CC and reboot all nodes. self.setup_for_test(skip_data_loading=True) self.ingestion_in_progress() self.node_type = self.input.param('node_type','CC') replica_nodes_before_reboot = self.cbas_util.get_replicas_info(self.shell) replicas_before_reboot=len(self.cbas_util.get_replicas_info(self.shell)) if self.node_type == "CC": NodeHelper.reboot_server(self.cbas_node, self) elif self.node_type == "NC": for server in self.cbas_servers: NodeHelper.reboot_server(server, self) else: NodeHelper.reboot_server(self.cbas_node, self) for server in self.cbas_servers: NodeHelper.reboot_server(server, self) self.sleep(60) replica_nodes_after_reboot = self.cbas_util.get_replicas_info(self.shell) replicas_after_reboot=len(replica_nodes_after_reboot) self.assertTrue(replica_nodes_after_reboot == replica_nodes_before_reboot, "Replica nodes changed after reboot. Before: %s , After : %s" %(replica_nodes_before_reboot,replica_nodes_after_reboot)) self.assertTrue(replicas_after_reboot == replicas_before_reboot, "Number of Replica nodes changed after reboot. Before: %s , After : %s" %(replicas_before_reboot,replicas_after_reboot)) items_in_cbas_bucket = 0 start_time=time.time() while (items_in_cbas_bucket == 0 or items_in_cbas_bucket == -1) and time.time()<start_time+60: try: items_in_cbas_bucket, _ = self.cbas_util.get_num_items_in_cbas_dataset(self.cbas_dataset_name) except: pass self.sleep(1) query = "select count(*) from {0};".format(self.cbas_dataset_name) self.cbas_util._run_concurrent_queries(query,"immediate",100) if not self.cbas_util.validate_cbas_dataset_items_count(self.cbas_dataset_name,self.num_items*2): self.fail("No. of items in CBAS dataset do not match that in the CB bucket") for replica in replica_nodes_after_reboot: self.log.info("replica state during rebalance: %s"%replica['status']) self.assertEqual(replica['status'], "IN_SYNC","Replica state is incorrect: %s"%replica['status']) self.ingest_more_data()
def test_logging_configurations_are_restored_post_service_restarts(self): self.log.info("Add a cbas node") result = self.add_node(self.cbas_servers[0], services=["cbas"], rebalance=True) self.assertTrue(result, msg="Failed to add CBAS node") self.log.info("Delete all loggers") self.cbas_util.delete_all_loggers_on_cbas() self.log.info("Set the logging level using the json object") status, content, response = self.cbas_util.set_log_level_on_cbas( CbasLogging.DEFAULT_LOGGER_CONFIG_DICT) self.assertTrue(status, msg="Response status incorrect for SET request") self.log.info("Delete specific logger") logger_name = self.input.param("logger_name_to_delete", "com.couchbase.client.core.node") status, content, response = self.cbas_util.delete_specific_cbas_log_level( logger_name) self.assertTrue(status, msg="Status mismatch for DELETE") del CbasLogging.DEFAULT_LOGGER_CONFIG_DICT[logger_name] self.log.info("Update specific logger") logger_name = self.input.param("logger_name_to_update", "org.apache.hyracks") logger_level_to_update = self.input.param("logger_level_to_update", "FATAL") status, response, content = self.cbas_util.set_specific_log_level_on_cbas( logger_name, logger_level_to_update) self.assertTrue(status, msg="Status mismatch for SET") CbasLogging.DEFAULT_LOGGER_CONFIG_DICT[ logger_name] = logger_level_to_update self.log.info("Add a new logger") logger_name = self.input.param("logger_name_to_add", "org.apache.hyracks123") logger_level_to_add = self.input.param("logger_level_to_add", "ALL") status, response, content = self.cbas_util.set_specific_log_level_on_cbas( logger_name, logger_level_to_add) self.assertTrue(status, msg="Status mismatch for SET") CbasLogging.DEFAULT_LOGGER_CONFIG_DICT[ logger_name] = logger_level_to_add self.log.info("Verify logging configuration that we set on cbas Node") for name, level in CbasLogging.DEFAULT_LOGGER_CONFIG_DICT.items(): status, content, response = self.cbas_util.get_specific_cbas_log_level( name) self.assertTrue(status, msg="Response status incorrect for GET request") self.assertEquals(content, level, msg="Logger configuration mismatch for logger " + name) self.sleep( timeout=10, message= "Waiting for logger configuration to be copied across cbas nodes") self.log.info("Verify logging configuration on other cbas node") for name, level in CbasLogging.DEFAULT_LOGGER_CONFIG_DICT.items(): status, content, response = cbas_utils( self.master, self.cbas_servers[0]).get_specific_cbas_log_level(name) self.assertTrue(status, msg="Response status incorrect for GET request") self.assertEquals(content, level, msg="Logger configuration mismatch for logger " + name) self.log.info("Read input params") process_name = self.input.param('process_name', None) service_name = self.input.param('service_name', None) restart_couchbase = self.input.param('restart_couchbase', False) reboot = self.input.param('reboot', False) kill_services = self.input.param('kill_services', False) self.log.info("Establish a remote connection") shell_cc = RemoteMachineShellConnection(self.cbas_node) shell_nc = RemoteMachineShellConnection(self.cbas_servers[0]) if kill_services: self.log.info("Kill the %s service on CC cbas node" % service_name) shell_cc.kill_process(process_name, service_name) self.log.info("Kill the %s service on other cbas node" % service_name) shell_nc.kill_process(process_name, service_name) if restart_couchbase: self.log.info("Restart couchbase CC node ") shell_cc.restart_couchbase() self.log.info("Restart couchbase NC node ") shell_nc.restart_couchbase() if reboot: self.log.info("Reboot couchbase CC node") NodeHelper.reboot_server(self.cbas_node, self) self.log.info("Reboot couchbase NC node") NodeHelper.reboot_server(self.cbas_servers[0], self) end_time = datetime.datetime.now() + datetime.timedelta(minutes=int(1)) self.log.info( "Wait for nodes to be bootstrapped, neglect the unreachable server exceptions" ) while datetime.datetime.now() < end_time: try: self.log.info("Get the logging configurations") status, content, response = self.cbas_util.get_log_level_on_cbas( ) self.assertTrue( status, msg="Response status incorrect for GET request") self.log.info("Convert response to a dictionary") log_dict = CbasLogging.convert_logger_get_result_to_a_dict( content) if len(log_dict) >= len( CbasLogging.DEFAULT_LOGGER_CONFIG_DICT): break except Exception as e: pass self.log.info("Verify logging configuration post service kill") for name, level in CbasLogging.DEFAULT_LOGGER_CONFIG_DICT.items(): status, content, response = self.cbas_util.get_specific_cbas_log_level( name) self.assertTrue(status, msg="Response status incorrect for GET request") self.assertEquals(content, level, msg="Logger configuration mismatch for logger " + name) self.sleep( timeout=10, message= "Waiting for logger configuration to be copied across cbas nodes") self.log.info( "Verify logging configuration on other cbas node post service kill" ) for name, level in CbasLogging.DEFAULT_LOGGER_CONFIG_DICT.items(): status, content, response = cbas_utils( self.master, self.cbas_servers[0]).get_specific_cbas_log_level(name) self.assertTrue(status, msg="Response status incorrect for GET request") self.assertEquals(content, level, msg="Logger configuration mismatch for logger " + name)
def test_logging_configurations_are_restored_post_service_restarts(self): self.log.info("Add a cbas node") result = self.add_node(self.cbas_servers[0], services=["cbas"], rebalance=True) self.assertTrue(result, msg="Failed to add CBAS node") self.log.info("Delete all loggers") self.cbas_util.delete_all_loggers_on_cbas() self.log.info("Set the logging level using the json object") status, content, response = self.cbas_util.set_log_level_on_cbas( CbasLogging.DEFAULT_LOGGER_CONFIG_DICT) self.assertTrue(status, msg="Response status incorrect for SET request") self.log.info("Delete specific logger") logger_name = self.input.param("logger_name_to_delete", "com.couchbase.client.core.node") status, content, response = self.cbas_util.delete_specific_cbas_log_level( logger_name) self.assertTrue(status, msg="Status mismatch for DELETE") del CbasLogging.DEFAULT_LOGGER_CONFIG_DICT[logger_name] self.log.info("Update specific logger") logger_name = self.input.param("logger_name_to_update", "org.apache.hyracks") logger_level_to_update = self.input.param("logger_level_to_update", "FATAL") status, response, content = self.cbas_util.set_specific_log_level_on_cbas( logger_name, logger_level_to_update) self.assertTrue(status, msg="Status mismatch for SET") CbasLogging.DEFAULT_LOGGER_CONFIG_DICT[ logger_name] = logger_level_to_update self.log.info("Add a new logger") logger_name = self.input.param("logger_name_to_add", "org.apache.hyracks123") logger_level_to_add = self.input.param("logger_level_to_add", "ALL") status, response, content = self.cbas_util.set_specific_log_level_on_cbas( logger_name, logger_level_to_add) self.assertTrue(status, msg="Status mismatch for SET") CbasLogging.DEFAULT_LOGGER_CONFIG_DICT[ logger_name] = logger_level_to_add self.log.info("Verify logging configuration that we set on cbas Node") for name, level in CbasLogging.DEFAULT_LOGGER_CONFIG_DICT.items(): status, content, response = self.cbas_util.get_specific_cbas_log_level( name) self.assertTrue(status, msg="Response status incorrect for GET request") self.assertEquals(content, level, msg="Logger configuration mismatch for logger " + name) self.sleep( timeout=10, message= "Waiting for logger configuration to be copied across cbas nodes") self.log.info("Verify logging configuration on other cbas node") for name, level in CbasLogging.DEFAULT_LOGGER_CONFIG_DICT.items(): status, content, response = cbas_utils( self.master, self.cbas_servers[0]).get_specific_cbas_log_level(name) self.assertTrue(status, msg="Response status incorrect for GET request") self.assertEquals(content, level, msg="Logger configuration mismatch for logger " + name) self.log.info("Read input params") process_name = self.input.param('process_name', None) service_name = self.input.param('service_name', None) restart_couchbase = self.input.param('restart_couchbase', False) reboot = self.input.param('reboot', False) kill_services = self.input.param('kill_services', False) self.log.info("Establish a remote connection") shell_cc = RemoteMachineShellConnection(self.cbas_node) shell_nc = RemoteMachineShellConnection(self.cbas_servers[0]) if kill_services: self.log.info("Kill the %s service on CC cbas node" % service_name) shell_cc.kill_process(process_name, service_name) self.log.info("Kill the %s service on other cbas node" % service_name) shell_nc.kill_process(process_name, service_name) if restart_couchbase: self.log.info("Restart couchbase CC node ") shell_cc.restart_couchbase() self.log.info("Restart couchbase NC node ") shell_nc.restart_couchbase() if reboot: self.log.info("Reboot couchbase CC node") NodeHelper.reboot_server(self.cbas_node, self) self.log.info("Reboot couchbase NC node") NodeHelper.reboot_server(self.cbas_servers[0], self) self.log.info( "Wait for request to complete and cluster to be active: Using private ping() function" ) cluster_recover_start_time = time.time() while time.time() < cluster_recover_start_time + 180: try: status, metrics, _, cbas_result, _ = self.cbas_util.execute_statement_on_cbas_util( "set `import-private-functions` `true`;ping();") if status == "success": break except: self.sleep(2, message="Wait for service to up again") self.log.info("Verify logging configuration post service kill") for name, level in CbasLogging.DEFAULT_LOGGER_CONFIG_DICT.items(): status, content, response = self.cbas_util.get_specific_cbas_log_level( name) self.assertTrue(status, msg="Response status incorrect for GET request") self.assertEquals(content, level, msg="Logger configuration mismatch for logger " + name) self.sleep( timeout=10, message= "Waiting for logger configuration to be copied across cbas nodes") self.log.info( "Verify logging configuration on other cbas node post service kill" ) for name, level in CbasLogging.DEFAULT_LOGGER_CONFIG_DICT.items(): status, content, response = cbas_utils( self.master, self.cbas_servers[0]).get_specific_cbas_log_level(name) self.assertTrue(status, msg="Response status incorrect for GET request") self.assertEquals(content, level, msg="Logger configuration mismatch for logger " + name)