def fts_rebalance_in(self): services_in = [] self.perform_doc_ops_in_all_cb_buckets(10000, 'create') time.sleep(30) for service in self.services_in.split("-"): services_in.append(service.split(":")[0]) self.log.info( "list of services to be added after formatting {0}".format( services_in)) servs_inout = self.servers[self.nodes_init:] rebalance = self.cluster.async_rebalance( self.servers[:self.nodes_init], servs_inout, [], services=services_in) rebalance.result() if self.ntonencrypt == 'disable' and self.enable_nton_local == True: ntonencryptionBase().setup_nton_cluster( self.servers, 'enable', self.local_clusterEncryption) self.create_fts_index_query_compare() self.sleep(10) #ntonencryptionBase().get_ntonencryption_status(self.servers) final_result = ntonencryptionBase().check_server_ports(self.servers) if self.ntonencrypt == 'disable' and self.enable_nton_local == True: result = ntonencryptionBase().validate_results( self.servers, final_result, self.local_clusterEncryption) else: result = ntonencryptionBase().validate_results( self.servers, final_result, self.ntonencrypt_level) self.log.info("Final result is {0}".format(final_result)) self.assertTrue(result)
def data_rebalance_in(self): ''' 1. Enable encryption on all nodes 2. Add node to current cluster ''' services_in = [] self.perform_doc_ops_in_all_cb_buckets(100, 'create') time.sleep(30) for service in self.services_in.split("-"): services_in.append(service.split(":")[0]) servs_inout = self.servers[self.nodes_init:] self.log.info("{0}".format(servs_inout)) rebalance = self.cluster.async_rebalance( self.servers[:self.nodes_init], servs_inout, [], services=services_in) rebalance.result() if self.ntonencrypt == 'disable' and self.enable_nton_local == True: ntonencryptionBase().setup_nton_cluster( self.servers, 'enable', self.local_clusterEncryption) #ntonencryptionBase(). (self.servers) final_result = ntonencryptionBase().check_server_ports(self.servers) result = ntonencryptionBase().validate_results(self.servers, final_result, self.ntonencrypt_level) self.log.info("Final result is {0}".format(final_result)) self.assertTrue(result)
def test_add_nodes_x509_rebalance(self): servs_inout = self.servers[self.nodes_init:] services_in = [] rest = RestConnection(self.master) copy_servers = copy.deepcopy(self.servers) self.log.info('before cert generate') x509main(self.master)._generate_cert(copy_servers, type='openssl', encryption='', key_length=1024, client_ip='172.16.1.174', alt_names='non_default', dns=None, uri=None, wildcard_dns=self.wildcard_dns) x509main(self.master).setup_master() x509main().setup_cluster_nodes_ssl(servs_inout) for service in self.services_in.split("-"): services_in.append(service.split(":")[0]) self.log.info( "list of services to be added after formatting {0}".format( services_in)) # add nodes to the cluster servs_inout = self.servers[1:4] rebalance = self.cluster.async_rebalance( self.servers[:self.nodes_init], servs_inout, [], services=services_in) rebalance.result() #Check if n2n can be enabled after adding x509 certificates if self.x509enable and self.ntonencrypt == 'enable': encryption_result = ntonencryptionBase().setup_nton_cluster( self.servers, 'enable', self.ntonencrypt_level) self.assertTrue(encryption_result, "Retries Exceeded. Cannot enable n2n encryption") self.check_all_services(self.servers) #ntonencryptionBase().get_ntonencryption_status(self.servers) #ntonencryptionBase().get_cluster_nton_status(self.master) final_result = ntonencryptionBase().check_server_ports(self.servers) result = ntonencryptionBase().validate_results(self.servers, final_result, self.ntonencrypt_level) self.assertTrue(result, 'Issue with results with x509 enable for sets') ntonencryptionBase().change_cluster_encryption_cli( self.servers, 'control') ntonencryptionBase().ntonencryption_cli(self.servers, 'disable') final_result_disable = ntonencryptionBase().check_server_ports( self.servers) self.log.info("{0}".format(final_result_disable)) result = ntonencryptionBase().validate_results(self.servers, final_result_disable, self.ntonencrypt_level, 'disable') self.assertTrue(result, 'Issue with results with x509 disable for sets')
def all_services_rebalance_in(self): self.perform_doc_ops_in_all_cb_buckets(1000, 'create', end_key=1000) time.sleep(30) services_in = [] self.log.info("list of services to be added {0}".format( self.services_in)) for service in self.services_in.split("-"): services_in.append(service.split(":")[0]) self.log.info( "list of services to be added after formatting {0}".format( services_in)) # add nodes to the cluster servs_inout = self.servers[self.nodes_init:] rebalance = self.cluster.async_rebalance( self.servers[:self.nodes_init], servs_inout, [], services=services_in) rebalance.result() if self.ntonencrypt == 'disable' and self.enable_nton_local == True: ntonencryptionBase().setup_nton_cluster( self.servers, 'enable', self.local_clusterEncryption) #ntonencryptionBase().get_ntonencryption_status(self.servers) cbas_node = self.get_nodes_from_services_map(service_type="cbas") if cbas_node is not None: self.create_cbas_index_query() if self.ntonencrypt == 'disable' and self.enable_nton_local == True: self.create_secondary_index_query('id') self.create_secondary_index_query('name', 'id', 'after') else: self.create_secondary_index_query('name') #ntonencryptionBase().get_ntonencryption_status(self.servers) if self.get_nodes_from_services_map(service_type="fts") is not None: self.create_fts_index_query_compare() self.perform_doc_ops_in_all_cb_buckets(2000, 'create', end_key=2000) #ntonencryptionBase().get_ntonencryption_status(self.servers) final_result = ntonencryptionBase().check_server_ports(self.servers) if self.ntonencrypt == 'disable' and self.enable_nton_local == True: result = ntonencryptionBase().validate_results( self.servers, final_result, self.local_clusterEncryption) else: result = ntonencryptionBase().validate_results( self.servers, final_result, self.ntonencrypt_level) self.log.info("Final result is {0}".format(final_result)) self.assertTrue(result)
def setUp(self): super(ntonencryptionTest, self).setUp() self.bucket_list = RestConnection(self.master).get_buckets() self.shell = RemoteMachineShellConnection(self.master) self.item_flag = self.input.param("item_flag", 4042322160) self.full_docs_list = '' self.n1ql_port = 8093 self.enable_nton_local = self.input.param('enable_nton_local',False) self.local_clusterEncryption = self.input.param('local_clusterEncryption','control') self.hostname = self.input.param('hostname',False) self.x509enable = self.input.param("x509enable",False) self.wildcard_dns = self.input.param("wildcard_dns",None) if self.ntonencrypt == 'enable' and not self.x509enable: ntonencryptionBase().setup_nton_cluster(self.servers,clusterEncryptionLevel=self.ntonencrypt_level)
def test_xdcr_with_security(self): #Settings self.settings_values_map = { "autofailover": ["enable", None], "n2n": ["enable", "disable"], "tls": ["all", "control", "strict"] } self.apply_settings_before_setup = self._input.param( "apply_settings_before_setup", False) self.disable_autofailover = self._input.param("disable_autofailover", False) self.enable_n2n = self._input.param("enable_n2n", False) self.enforce_tls = self._input.param("enforce_tls", None) self.tls_level = self._input.param("tls_level", "control") self.enable_autofailover = self._input.param("enable_autofailover", False) self.disable_n2n = self._input.param("disable_n2n", None) self.disable_tls = self._input.param("disable_tls", None) rebalance_in = self._input.param("rebalance_in", None) rebalance_out = self._input.param("rebalance_out", None) swap_rebalance = self._input.param("swap_rebalance", None) failover = self._input.param("failover", None) graceful = self._input.param("graceful", None) pause = self._input.param("pause", None) reboot = self._input.param("reboot", None) initial_xdcr = self._input.param("initial_xdcr", random.choice([True, False])) random_setting = self._input.param("random_setting", False) multiple_ca = self._input.param("multiple_ca", None) use_client_certs = self._input.param("use_client_certs", None) int_ca_name = self._input.param("int_ca_name", "iclient1_clientroot") all_node_upload = self._input.param("all_node_upload", False) rotate_certs = self._input.param("rotate_certs", None) delete_certs = self._input.param("delete_certs", None) restart_pkey_nodes = self._input.param("restart_pkey_nodes", None) if not self.apply_settings_before_setup: if initial_xdcr: self.load_and_setup_xdcr() else: self.setup_xdcr_and_load() if self.enforce_tls: for cluster in self.get_cluster_objects_for_input( self.enforce_tls): if self.tls_level == "rotate": for level in self.settings_values_map["tls"]: cluster.toggle_security_setting( [cluster.get_master_node()], "tls", level) time.sleep(5) else: cluster.toggle_security_setting( [cluster.get_master_node()], "tls", self.tls_level) #Revert to default (control) tls level if self.disable_tls: for cluster in self.get_cluster_objects_for_input( self.disable_tls): cluster.toggle_security_setting([cluster.get_master_node()], "tls") if self.enable_n2n: for cluster in self.get_cluster_objects_for_input(self.enable_n2n): cluster.toggle_security_setting([cluster.get_master_node()], "n2n", "enable") if self.disable_n2n: for cluster in self.get_cluster_objects_for_input( self.disable_n2n): cluster.toggle_security_setting([cluster.get_master_node()], "n2n") if self.enable_autofailover: for cluster in self.get_cluster_objects_for_input( self.enable_autofailover): cluster.toggle_security_setting([cluster.get_master_node()], "autofailover", "enable") if self.disable_autofailover: for cluster in self.get_cluster_objects_for_input( self.disable_autofailover): cluster.toggle_security_setting([cluster.get_master_node()], "autofailover") if random_setting: for cluster in self.get_cluster_objects_for_input(random_setting): setting = random.choice(list(self.settings_values_map.keys())) value = random.choice(self.settings_values_map.get(setting)) cluster.toggle_security_setting([cluster.get_master_node()], setting, value) if multiple_ca: for cluster in self.get_cluster_objects_for_input(multiple_ca): master = cluster.get_master_node() ntonencryptionBase().disable_nton_cluster([master]) CbServer.x509 = x509main(host=master) for server in cluster.get_nodes(): CbServer.x509.delete_inbox_folder_on_server(server=server) CbServer.x509.generate_multiple_x509_certs( servers=cluster.get_nodes()) if all_node_upload: for node_num in range(len(cluster.get_nodes())): CbServer.x509.upload_root_certs( server=cluster.get_nodes()[node_num], root_ca_names=[ CbServer.x509.root_ca_names[node_num] ]) else: for server in cluster.get_nodes(): CbServer.x509.upload_root_certs(server) CbServer.x509.upload_node_certs(servers=cluster.get_nodes()) if use_client_certs: CbServer.x509.upload_client_cert_settings(server=master) client_cert_path, client_key_path = CbServer.x509.get_client_cert( int_ca_name=int_ca_name) # Copy the certs onto the test machines for server in cluster.get_nodes(): shell = RemoteMachineShellConnection(server) shell.execute_command( f"mkdir -p {os.path.dirname(client_cert_path)}") shell.copy_file_local_to_remote( client_cert_path, client_cert_path) shell.execute_command( f"mkdir -p {CbServer.x509.CACERTFILEPATH}all") shell.copy_file_local_to_remote( f"{CbServer.x509.CACERTFILEPATH}all/all_ca.pem", f"{CbServer.x509.CACERTFILEPATH}all/all_ca.pem") shell.disconnect() self._client_cert = self._read_from_file(client_cert_path) self._client_key = self._read_from_file(client_key_path) self.add_built_in_server_user(node=master) ntonencryptionBase().setup_nton_cluster( [master], clusterEncryptionLevel="strict") if rotate_certs: for cluster in self.get_cluster_objects_for_input( rotate_certs): CbServer.x509.rotate_certs(cluster.get_nodes()) if delete_certs: for cluster in self.get_cluster_objects_for_input( delete_certs): for node in cluster.get_nodes(): CbServer.x509.delete_trusted_CAs(node) if restart_pkey_nodes: for cluster in self.get_cluster_objects_for_input( restart_pkey_nodes): for node in cluster.get_nodes(): shell = RemoteMachineShellConnection(node) shell.restart_couchbase() shell.disconnect() time.sleep(10) cluster.failover_and_rebalance_nodes() cluster.add_back_node("delta") if self.apply_settings_before_setup: if initial_xdcr: self.load_and_setup_xdcr() else: self.setup_xdcr_and_load() if pause: for cluster in self.get_cluster_objects_for_input(pause): for remote_cluster_refs in cluster.get_remote_clusters(): remote_cluster_refs.pause_all_replications() time.sleep(60) if rebalance_in: for cluster in self.get_cluster_objects_for_input(rebalance_in): cluster.rebalance_in() if failover: for cluster in self.get_cluster_objects_for_input(failover): cluster.failover_and_rebalance_nodes(graceful=graceful, rebalance=True) if rebalance_out: for cluster in self.get_cluster_objects_for_input(rebalance_out): cluster.rebalance_out() if swap_rebalance: for cluster in self.get_cluster_objects_for_input(swap_rebalance): cluster.swap_rebalance() if pause: for cluster in self.get_cluster_objects_for_input(pause): for remote_cluster_refs in cluster.get_remote_clusters(): remote_cluster_refs.resume_all_replications() if reboot: for cluster in self.get_cluster_objects_for_input(reboot): cluster.warmup_node() time.sleep(60) self.perform_update_delete() self.verify_results()
def test_init_nodes_x509(self): servs_inout = self.servers[1:4] rest = RestConnection(self.master) copy_servers = copy.deepcopy(self.servers) self.log.info('before cert generate') if self.x509enable: x509main(self.master)._generate_cert( copy_servers, type='openssl', encryption='', key_length=1024, client_ip='172.16.1.174', alt_names='non_default', dns=None, uri=None, wildcard_dns=self.wildcard_dns) x509main(self.master).setup_master() x509main().setup_cluster_nodes_ssl(servs_inout, True) if self.ntonencrypt == 'disable' and self.enable_nton_local == True: ntonencryptionBase().setup_nton_cluster( self.servers, 'enable', self.local_clusterEncryption) # Check if n2n can be enabled after adding x509 certificates elif self.x509enable and self.ntonencrypt == 'enable': encryption_result = ntonencryptionBase().setup_nton_cluster( self.servers, 'enable', self.ntonencrypt_level) self.assertTrue(encryption_result, "Retries Exceeded. Cannot enable n2n encryption") self.check_all_services(self.servers) #ntonencryptionBase().get_ntonencryption_status(self.servers) #ntonencryptionBase().get_cluster_nton_status(self.master) final_result = ntonencryptionBase().check_server_ports(self.servers) self.log.info("{0}".format(final_result)) result = ntonencryptionBase().validate_results(self.servers, final_result, self.ntonencrypt_level) self.assertTrue(result, 'Issue with results with x509 enable for sets') ntonencryptionBase().change_cluster_encryption_cli( self.servers, 'control') ntonencryptionBase().ntonencryption_cli(self.servers, 'disable') final_result_disable = ntonencryptionBase().check_server_ports( self.servers) result = ntonencryptionBase().validate_results(self.servers, final_result_disable, self.ntonencrypt_level, 'disable') self.assertTrue(result, 'Issue with results with x509 disable for sets')
def tearDown(self): super(ntonencryptionTest, self).tearDown() ntonencryptionBase().disable_nton_cluster(self.servers) self._reset_original()
def test_create_and_alter_index_with_nodeinfo_on_all_encryption_mode(self): ''' Enable node to node encryption and set encryption level to all. Validate that create/alter indexes work with both 8091 and 18091 ports. https://issues.couchbase.com/browse/MB-49879 ''' ntonencryptionBase().setup_nton_cluster([self.master], clusterEncryptionLevel="all") index_nodes = self.get_nodes_from_services_map(service_type="index", get_all_nodes=True) if len(index_nodes) < 2: self.skipTest("Can't run Alter index tests with less than 2 Index nodes") self.prepare_collection_for_indexing() collection_namespace = self.namespaces[0] servers = self.get_nodes_from_services_map(service_type="index", get_all_nodes=True) host1, host2, port1, port2 = servers[0].ip, servers[1].ip, servers[0].port, 18091 # create index on port 8091 on node1 and 18091 on node 2 index_gen1 = QueryDefinition(index_name='idx1', index_fields=['age']) deploy_node_info = ["{0}:{1}".format(host1, port1)] query = index_gen1.generate_index_create_query(namespace=collection_namespace, deploy_node_info=deploy_node_info) index_gen2 = QueryDefinition(index_name='idx2', index_fields=['city']) deploy_node_info2 = ["{0}:{1}".format(host2, port2)] query2 = index_gen2.generate_index_create_query(namespace=collection_namespace, deploy_node_info=deploy_node_info2) self.run_cbq_query(query=query) self.run_cbq_query(query=query2) self.wait_until_indexes_online() self.sleep(10) index_info = self.get_index_map()[self.test_bucket] for idx in index_info: if idx == 'idx1' and host1 not in index_info[idx]['hosts']: self.fail(f"Index idx1 not on expected host. Expected host {host1} Actual host {index_info[idx]['hosts']}") elif idx == 'idx2' and host2 not in index_info[idx]['hosts']: self.fail(f"Index idx2 not on expected host. Expected host {host2} Actual host {index_info[idx]['hosts']}") # Verify if the newly created indexes are being used with an explain query query = f'explain select count(*) from {collection_namespace} where age > 45' result = self.run_cbq_query(query=query)['results'] self.assertEqual(result[0]['plan']['~children'][0]['index'], 'idx1', "idx1 index is not used for age scan.") query = f'explain select count(*) from {collection_namespace} where city like "a%"' result = self.run_cbq_query(query=query)['results'] self.assertEqual(result[0]['plan']['~children'][0]['index'], 'idx2', "idx2 index is not used for city scan.") # alter idx1 on 18091 and increase replica count nodes_info1 = ["{0}:18091".format(server.ip,server.port) for server in servers[:2]] self.alter_index_replicas(index_name='idx1', namespace=collection_namespace, action='replica_count', nodes=nodes_info1, num_replicas=1) # alter idx2 on 8091 and increase replica count nodes = ["{0}:{1}".format(server.ip, server.port) for server in servers[:2]] self.alter_index_replicas(index_name='idx2', namespace=collection_namespace, action='replica_count', nodes=nodes, num_replicas=1) self.sleep(10) self.wait_until_indexes_online() index_info = self.get_index_map()[self.test_bucket] # verify that index hosts info gets updated for the replica hosts for idx in index_info: if 'idx1' in idx: if 'replica' not in idx: if host1 not in index_info[idx]['hosts']: self.fail(f"Index idx1 not on expected host after alter statement. " f"Expected host {host1} Actual host {index_info[idx]['hosts']}") else: if host2 not in index_info[idx]['hosts']: self.fail(f"Index idx1 (replica) not on expected host after alter statement" f". Expected host {host2} Actual host {index_info[idx]['hosts']}") elif idx == 'idx2': if 'replica' not in idx: if host2 not in index_info[idx]['hosts']: self.fail(f"Index idx2 replica not on expected host after alter statement. " f"Expected host {host2} Actual host {index_info[idx]['hosts']}") else: if host1 not in index_info[idx]['hosts']: self.fail(f"Index idx2 not on expected host after alter statement" f". Expected host {host1} Actual host {index_info[idx]['hosts']}")