def fts_rebalance_in(self):
services_in = []
self.perform_doc_ops_in_all_cb_buckets(10000, 'create')
time.sleep(30)
for service in self.services_in.split("-"):
services_in.append(service.split(":")[0])
self.log.info(
"list of services to be added after formatting {0}".format(
services_in))
servs_inout = self.servers[self.nodes_init:]
rebalance = self.cluster.async_rebalance(
self.servers[:self.nodes_init],
servs_inout, [],
services=services_in)
rebalance.result()
if self.ntonencrypt == 'disable' and self.enable_nton_local == True:
ntonencryptionBase().setup_nton_cluster(
self.servers, 'enable', self.local_clusterEncryption)
self.create_fts_index_query_compare()
self.sleep(10)
#ntonencryptionBase().get_ntonencryption_status(self.servers)
final_result = ntonencryptionBase().check_server_ports(self.servers)
if self.ntonencrypt == 'disable' and self.enable_nton_local == True:
result = ntonencryptionBase().validate_results(
self.servers, final_result, self.local_clusterEncryption)
else:
result = ntonencryptionBase().validate_results(
self.servers, final_result, self.ntonencrypt_level)
self.log.info("Final result is {0}".format(final_result))
self.assertTrue(result)
def data_rebalance_in(self):
'''
1. Enable encryption on all nodes
2. Add node to current cluster
'''
services_in = []
self.perform_doc_ops_in_all_cb_buckets(100, 'create')
time.sleep(30)
for service in self.services_in.split("-"):
services_in.append(service.split(":")[0])
servs_inout = self.servers[self.nodes_init:]
self.log.info("{0}".format(servs_inout))
rebalance = self.cluster.async_rebalance(
self.servers[:self.nodes_init],
servs_inout, [],
services=services_in)
rebalance.result()
if self.ntonencrypt == 'disable' and self.enable_nton_local == True:
ntonencryptionBase().setup_nton_cluster(
self.servers, 'enable', self.local_clusterEncryption)
#ntonencryptionBase(). (self.servers)
final_result = ntonencryptionBase().check_server_ports(self.servers)
result = ntonencryptionBase().validate_results(self.servers,
final_result,
self.ntonencrypt_level)
self.log.info("Final result is {0}".format(final_result))
self.assertTrue(result)
def test_add_nodes_x509_rebalance(self):
servs_inout = self.servers[self.nodes_init:]
services_in = []
rest = RestConnection(self.master)
copy_servers = copy.deepcopy(self.servers)
self.log.info('before cert generate')
x509main(self.master)._generate_cert(copy_servers,
type='openssl',
encryption='',
key_length=1024,
client_ip='172.16.1.174',
alt_names='non_default',
dns=None,
uri=None,
wildcard_dns=self.wildcard_dns)
x509main(self.master).setup_master()
x509main().setup_cluster_nodes_ssl(servs_inout)
for service in self.services_in.split("-"):
services_in.append(service.split(":")[0])
self.log.info(
"list of services to be added after formatting {0}".format(
services_in))
# add nodes to the cluster
servs_inout = self.servers[1:4]
rebalance = self.cluster.async_rebalance(
self.servers[:self.nodes_init],
servs_inout, [],
services=services_in)
rebalance.result()
#Check if n2n can be enabled after adding x509 certificates
if self.x509enable and self.ntonencrypt == 'enable':
encryption_result = ntonencryptionBase().setup_nton_cluster(
self.servers, 'enable', self.ntonencrypt_level)
self.assertTrue(encryption_result,
"Retries Exceeded. Cannot enable n2n encryption")
self.check_all_services(self.servers)
#ntonencryptionBase().get_ntonencryption_status(self.servers)
#ntonencryptionBase().get_cluster_nton_status(self.master)
final_result = ntonencryptionBase().check_server_ports(self.servers)
result = ntonencryptionBase().validate_results(self.servers,
final_result,
self.ntonencrypt_level)
self.assertTrue(result, 'Issue with results with x509 enable for sets')
ntonencryptionBase().change_cluster_encryption_cli(
self.servers, 'control')
ntonencryptionBase().ntonencryption_cli(self.servers, 'disable')
final_result_disable = ntonencryptionBase().check_server_ports(
self.servers)
self.log.info("{0}".format(final_result_disable))
result = ntonencryptionBase().validate_results(self.servers,
final_result_disable,
self.ntonencrypt_level,
'disable')
self.assertTrue(result,
'Issue with results with x509 disable for sets')
def all_services_rebalance_in(self):
self.perform_doc_ops_in_all_cb_buckets(1000, 'create', end_key=1000)
time.sleep(30)
services_in = []
self.log.info("list of services to be added {0}".format(
self.services_in))
for service in self.services_in.split("-"):
services_in.append(service.split(":")[0])
self.log.info(
"list of services to be added after formatting {0}".format(
services_in))
# add nodes to the cluster
servs_inout = self.servers[self.nodes_init:]
rebalance = self.cluster.async_rebalance(
self.servers[:self.nodes_init],
servs_inout, [],
services=services_in)
rebalance.result()
if self.ntonencrypt == 'disable' and self.enable_nton_local == True:
ntonencryptionBase().setup_nton_cluster(
self.servers, 'enable', self.local_clusterEncryption)
#ntonencryptionBase().get_ntonencryption_status(self.servers)
cbas_node = self.get_nodes_from_services_map(service_type="cbas")
if cbas_node is not None:
self.create_cbas_index_query()
if self.ntonencrypt == 'disable' and self.enable_nton_local == True:
self.create_secondary_index_query('id')
self.create_secondary_index_query('name', 'id', 'after')
else:
self.create_secondary_index_query('name')
#ntonencryptionBase().get_ntonencryption_status(self.servers)
if self.get_nodes_from_services_map(service_type="fts") is not None:
self.create_fts_index_query_compare()
self.perform_doc_ops_in_all_cb_buckets(2000, 'create', end_key=2000)
#ntonencryptionBase().get_ntonencryption_status(self.servers)
final_result = ntonencryptionBase().check_server_ports(self.servers)
if self.ntonencrypt == 'disable' and self.enable_nton_local == True:
result = ntonencryptionBase().validate_results(
self.servers, final_result, self.local_clusterEncryption)
else:
result = ntonencryptionBase().validate_results(
self.servers, final_result, self.ntonencrypt_level)
self.log.info("Final result is {0}".format(final_result))
self.assertTrue(result)
def setUp(self):
super(ntonencryptionTest, self).setUp()
self.bucket_list = RestConnection(self.master).get_buckets()
self.shell = RemoteMachineShellConnection(self.master)
self.item_flag = self.input.param("item_flag", 4042322160)
self.full_docs_list = ''
self.n1ql_port = 8093
self.enable_nton_local = self.input.param('enable_nton_local',False)
self.local_clusterEncryption = self.input.param('local_clusterEncryption','control')
self.hostname = self.input.param('hostname',False)
self.x509enable = self.input.param("x509enable",False)
self.wildcard_dns = self.input.param("wildcard_dns",None)
if self.ntonencrypt == 'enable' and not self.x509enable:
ntonencryptionBase().setup_nton_cluster(self.servers,clusterEncryptionLevel=self.ntonencrypt_level)
def test_xdcr_with_security(self):
#Settings
self.settings_values_map = {
"autofailover": ["enable", None],
"n2n": ["enable", "disable"],
"tls": ["all", "control", "strict"]
}
self.apply_settings_before_setup = self._input.param(
"apply_settings_before_setup", False)
self.disable_autofailover = self._input.param("disable_autofailover",
False)
self.enable_n2n = self._input.param("enable_n2n", False)
self.enforce_tls = self._input.param("enforce_tls", None)
self.tls_level = self._input.param("tls_level", "control")
self.enable_autofailover = self._input.param("enable_autofailover",
False)
self.disable_n2n = self._input.param("disable_n2n", None)
self.disable_tls = self._input.param("disable_tls", None)
rebalance_in = self._input.param("rebalance_in", None)
rebalance_out = self._input.param("rebalance_out", None)
swap_rebalance = self._input.param("swap_rebalance", None)
failover = self._input.param("failover", None)
graceful = self._input.param("graceful", None)
pause = self._input.param("pause", None)
reboot = self._input.param("reboot", None)
initial_xdcr = self._input.param("initial_xdcr",
random.choice([True, False]))
random_setting = self._input.param("random_setting", False)
multiple_ca = self._input.param("multiple_ca", None)
use_client_certs = self._input.param("use_client_certs", None)
int_ca_name = self._input.param("int_ca_name", "iclient1_clientroot")
all_node_upload = self._input.param("all_node_upload", False)
rotate_certs = self._input.param("rotate_certs", None)
delete_certs = self._input.param("delete_certs", None)
restart_pkey_nodes = self._input.param("restart_pkey_nodes", None)
if not self.apply_settings_before_setup:
if initial_xdcr:
self.load_and_setup_xdcr()
else:
self.setup_xdcr_and_load()
if self.enforce_tls:
for cluster in self.get_cluster_objects_for_input(
self.enforce_tls):
if self.tls_level == "rotate":
for level in self.settings_values_map["tls"]:
cluster.toggle_security_setting(
[cluster.get_master_node()], "tls", level)
time.sleep(5)
else:
cluster.toggle_security_setting(
[cluster.get_master_node()], "tls", self.tls_level)
#Revert to default (control) tls level
if self.disable_tls:
for cluster in self.get_cluster_objects_for_input(
self.disable_tls):
cluster.toggle_security_setting([cluster.get_master_node()],
"tls")
if self.enable_n2n:
for cluster in self.get_cluster_objects_for_input(self.enable_n2n):
cluster.toggle_security_setting([cluster.get_master_node()],
"n2n", "enable")
if self.disable_n2n:
for cluster in self.get_cluster_objects_for_input(
self.disable_n2n):
cluster.toggle_security_setting([cluster.get_master_node()],
"n2n")
if self.enable_autofailover:
for cluster in self.get_cluster_objects_for_input(
self.enable_autofailover):
cluster.toggle_security_setting([cluster.get_master_node()],
"autofailover", "enable")
if self.disable_autofailover:
for cluster in self.get_cluster_objects_for_input(
self.disable_autofailover):
cluster.toggle_security_setting([cluster.get_master_node()],
"autofailover")
if random_setting:
for cluster in self.get_cluster_objects_for_input(random_setting):
setting = random.choice(list(self.settings_values_map.keys()))
value = random.choice(self.settings_values_map.get(setting))
cluster.toggle_security_setting([cluster.get_master_node()],
setting, value)
if multiple_ca:
for cluster in self.get_cluster_objects_for_input(multiple_ca):
master = cluster.get_master_node()
ntonencryptionBase().disable_nton_cluster([master])
CbServer.x509 = x509main(host=master)
for server in cluster.get_nodes():
CbServer.x509.delete_inbox_folder_on_server(server=server)
CbServer.x509.generate_multiple_x509_certs(
servers=cluster.get_nodes())
if all_node_upload:
for node_num in range(len(cluster.get_nodes())):
CbServer.x509.upload_root_certs(
server=cluster.get_nodes()[node_num],
root_ca_names=[
CbServer.x509.root_ca_names[node_num]
])
else:
for server in cluster.get_nodes():
CbServer.x509.upload_root_certs(server)
CbServer.x509.upload_node_certs(servers=cluster.get_nodes())
if use_client_certs:
CbServer.x509.upload_client_cert_settings(server=master)
client_cert_path, client_key_path = CbServer.x509.get_client_cert(
int_ca_name=int_ca_name)
# Copy the certs onto the test machines
for server in cluster.get_nodes():
shell = RemoteMachineShellConnection(server)
shell.execute_command(
f"mkdir -p {os.path.dirname(client_cert_path)}")
shell.copy_file_local_to_remote(
client_cert_path, client_cert_path)
shell.execute_command(
f"mkdir -p {CbServer.x509.CACERTFILEPATH}all")
shell.copy_file_local_to_remote(
f"{CbServer.x509.CACERTFILEPATH}all/all_ca.pem",
f"{CbServer.x509.CACERTFILEPATH}all/all_ca.pem")
shell.disconnect()
self._client_cert = self._read_from_file(client_cert_path)
self._client_key = self._read_from_file(client_key_path)
self.add_built_in_server_user(node=master)
ntonencryptionBase().setup_nton_cluster(
[master], clusterEncryptionLevel="strict")
if rotate_certs:
for cluster in self.get_cluster_objects_for_input(
rotate_certs):
CbServer.x509.rotate_certs(cluster.get_nodes())
if delete_certs:
for cluster in self.get_cluster_objects_for_input(
delete_certs):
for node in cluster.get_nodes():
CbServer.x509.delete_trusted_CAs(node)
if restart_pkey_nodes:
for cluster in self.get_cluster_objects_for_input(
restart_pkey_nodes):
for node in cluster.get_nodes():
shell = RemoteMachineShellConnection(node)
shell.restart_couchbase()
shell.disconnect()
time.sleep(10)
cluster.failover_and_rebalance_nodes()
cluster.add_back_node("delta")
if self.apply_settings_before_setup:
if initial_xdcr:
self.load_and_setup_xdcr()
else:
self.setup_xdcr_and_load()
if pause:
for cluster in self.get_cluster_objects_for_input(pause):
for remote_cluster_refs in cluster.get_remote_clusters():
remote_cluster_refs.pause_all_replications()
time.sleep(60)
if rebalance_in:
for cluster in self.get_cluster_objects_for_input(rebalance_in):
cluster.rebalance_in()
if failover:
for cluster in self.get_cluster_objects_for_input(failover):
cluster.failover_and_rebalance_nodes(graceful=graceful,
rebalance=True)
if rebalance_out:
for cluster in self.get_cluster_objects_for_input(rebalance_out):
cluster.rebalance_out()
if swap_rebalance:
for cluster in self.get_cluster_objects_for_input(swap_rebalance):
cluster.swap_rebalance()
if pause:
for cluster in self.get_cluster_objects_for_input(pause):
for remote_cluster_refs in cluster.get_remote_clusters():
remote_cluster_refs.resume_all_replications()
if reboot:
for cluster in self.get_cluster_objects_for_input(reboot):
cluster.warmup_node()
time.sleep(60)
self.perform_update_delete()
self.verify_results()
def test_init_nodes_x509(self):
servs_inout = self.servers[1:4]
rest = RestConnection(self.master)
copy_servers = copy.deepcopy(self.servers)
self.log.info('before cert generate')
if self.x509enable:
x509main(self.master)._generate_cert(
copy_servers,
type='openssl',
encryption='',
key_length=1024,
client_ip='172.16.1.174',
alt_names='non_default',
dns=None,
uri=None,
wildcard_dns=self.wildcard_dns)
x509main(self.master).setup_master()
x509main().setup_cluster_nodes_ssl(servs_inout, True)
if self.ntonencrypt == 'disable' and self.enable_nton_local == True:
ntonencryptionBase().setup_nton_cluster(
self.servers, 'enable', self.local_clusterEncryption)
# Check if n2n can be enabled after adding x509 certificates
elif self.x509enable and self.ntonencrypt == 'enable':
encryption_result = ntonencryptionBase().setup_nton_cluster(
self.servers, 'enable', self.ntonencrypt_level)
self.assertTrue(encryption_result,
"Retries Exceeded. Cannot enable n2n encryption")
self.check_all_services(self.servers)
#ntonencryptionBase().get_ntonencryption_status(self.servers)
#ntonencryptionBase().get_cluster_nton_status(self.master)
final_result = ntonencryptionBase().check_server_ports(self.servers)
self.log.info("{0}".format(final_result))
result = ntonencryptionBase().validate_results(self.servers,
final_result,
self.ntonencrypt_level)
self.assertTrue(result, 'Issue with results with x509 enable for sets')
ntonencryptionBase().change_cluster_encryption_cli(
self.servers, 'control')
ntonencryptionBase().ntonencryption_cli(self.servers, 'disable')
final_result_disable = ntonencryptionBase().check_server_ports(
self.servers)
result = ntonencryptionBase().validate_results(self.servers,
final_result_disable,
self.ntonencrypt_level,
'disable')
self.assertTrue(result,
'Issue with results with x509 disable for sets')
def tearDown(self):
super(ntonencryptionTest, self).tearDown()
ntonencryptionBase().disable_nton_cluster(self.servers)
self._reset_original()
def test_create_and_alter_index_with_nodeinfo_on_all_encryption_mode(self):
'''
Enable node to node encryption and set encryption level to all.
Validate that create/alter indexes work with both 8091 and 18091 ports.
https://issues.couchbase.com/browse/MB-49879
'''
ntonencryptionBase().setup_nton_cluster([self.master], clusterEncryptionLevel="all")
index_nodes = self.get_nodes_from_services_map(service_type="index", get_all_nodes=True)
if len(index_nodes) < 2:
self.skipTest("Can't run Alter index tests with less than 2 Index nodes")
self.prepare_collection_for_indexing()
collection_namespace = self.namespaces[0]
servers = self.get_nodes_from_services_map(service_type="index", get_all_nodes=True)
host1, host2, port1, port2 = servers[0].ip, servers[1].ip, servers[0].port, 18091
# create index on port 8091 on node1 and 18091 on node 2
index_gen1 = QueryDefinition(index_name='idx1', index_fields=['age'])
deploy_node_info = ["{0}:{1}".format(host1, port1)]
query = index_gen1.generate_index_create_query(namespace=collection_namespace,
deploy_node_info=deploy_node_info)
index_gen2 = QueryDefinition(index_name='idx2', index_fields=['city'])
deploy_node_info2 = ["{0}:{1}".format(host2, port2)]
query2 = index_gen2.generate_index_create_query(namespace=collection_namespace, deploy_node_info=deploy_node_info2)
self.run_cbq_query(query=query)
self.run_cbq_query(query=query2)
self.wait_until_indexes_online()
self.sleep(10)
index_info = self.get_index_map()[self.test_bucket]
for idx in index_info:
if idx == 'idx1' and host1 not in index_info[idx]['hosts']:
self.fail(f"Index idx1 not on expected host. Expected host {host1} Actual host {index_info[idx]['hosts']}")
elif idx == 'idx2' and host2 not in index_info[idx]['hosts']:
self.fail(f"Index idx2 not on expected host. Expected host {host2} Actual host {index_info[idx]['hosts']}")
# Verify if the newly created indexes are being used with an explain query
query = f'explain select count(*) from {collection_namespace} where age > 45'
result = self.run_cbq_query(query=query)['results']
self.assertEqual(result[0]['plan']['~children'][0]['index'], 'idx1',
"idx1 index is not used for age scan.")
query = f'explain select count(*) from {collection_namespace} where city like "a%"'
result = self.run_cbq_query(query=query)['results']
self.assertEqual(result[0]['plan']['~children'][0]['index'], 'idx2',
"idx2 index is not used for city scan.")
# alter idx1 on 18091 and increase replica count
nodes_info1 = ["{0}:18091".format(server.ip,server.port) for server in servers[:2]]
self.alter_index_replicas(index_name='idx1', namespace=collection_namespace, action='replica_count',
nodes=nodes_info1, num_replicas=1)
# alter idx2 on 8091 and increase replica count
nodes = ["{0}:{1}".format(server.ip, server.port) for server in servers[:2]]
self.alter_index_replicas(index_name='idx2', namespace=collection_namespace, action='replica_count',
nodes=nodes, num_replicas=1)
self.sleep(10)
self.wait_until_indexes_online()
index_info = self.get_index_map()[self.test_bucket]
# verify that index hosts info gets updated for the replica hosts
for idx in index_info:
if 'idx1' in idx:
if 'replica' not in idx:
if host1 not in index_info[idx]['hosts']:
self.fail(f"Index idx1 not on expected host after alter statement. "
f"Expected host {host1} Actual host {index_info[idx]['hosts']}")
else:
if host2 not in index_info[idx]['hosts']:
self.fail(f"Index idx1 (replica) not on expected host after alter statement"
f". Expected host {host2} Actual host {index_info[idx]['hosts']}")
elif idx == 'idx2':
if 'replica' not in idx:
if host2 not in index_info[idx]['hosts']:
self.fail(f"Index idx2 replica not on expected host after alter statement. "
f"Expected host {host2} Actual host {index_info[idx]['hosts']}")
else:
if host1 not in index_info[idx]['hosts']:
self.fail(f"Index idx2 not on expected host after alter statement"
f". Expected host {host1} Actual host {index_info[idx]['hosts']}")
