def setUp(self):
super(MultipleCA, self).setUp()
self.standard = self.input.param("standard", "pkcs8")
self.passphrase_type = self.input.param("passphrase_type", "script")
self.encryption_type = self.input.param("encryption_type", "aes256")
self.wildcard_dns = self.input.param("wildcard_dns", None)
self.passphrase_url = self.input.param(
"rest_url", "https://testingsomething.free.beeceptor.com/")
self.x509 = x509main(host=self.master,
standard=self.standard,
encryption_type=self.encryption_type,
passphrase_type=self.passphrase_type,
wildcard_dns=self.wildcard_dns,
passhprase_url=self.passphrase_url)
for server in self.servers:
self.x509.delete_inbox_folder_on_server(server=server)
sample_bucket = self.input.param("sample_bucket", "travel-sample")
if sample_bucket is not None:
self.load_sample_bucket(self.master, sample_bucket)
self.buckets = RestConnection(self.master).get_buckets()
rest = RestConnection(self.master)
for bucket in self.buckets:
rest.change_bucket_props(bucket, replicaNumber=self.num_replicas)
task = self.cluster.async_rebalance(self.servers[:self.nodes_init], [],
[])
self.wait_for_rebalance_to_complete(task)
self.n2n_encryption_level_multiple_CA = self.input.param(
"n2n_encryption_level_multiple_CA", None)
if self.n2n_encryption_level_multiple_CA:
ntonencryptionBase().setup_nton_cluster(
[self.master],
clusterEncryptionLevel=self.n2n_encryption_level_multiple_CA)
CbServer.use_https = True
def tearDown(self):
if self.input.param("n2n_encryption_level_multiple_CA", None):
ntonencryptionBase().disable_nton_cluster([self.master])
CbServer.use_https = False
self.x509 = x509main(host=self.master)
self.x509.teardown_certs(servers=self.servers)
super(MultipleCA, self).tearDown()
def test_multiple_CA(self):
ops = self.input.param("ops", None)
self.x509 = x509main(host=self.master)
self.x509.generate_multiple_x509_certs(servers=self.servers)
if ops == "load_cluster_CA":
self.x509.upload_root_certs(server=self.master,
root_ca_names=["clientroot"])
content = self.x509.get_trusted_CAs(server=self.master)
for ca_dict in content:
subject = ca_dict["subject"]
root_ca_name = subject.split("CN=")[1]
if root_ca_name == "clientroot":
expires = ca_dict["notAfter"]
expectedResults = {
"description": "Upload cluster CA",
"expires": expires,
"local": {
"ip": self.master.ip,
"port": 8091
},
"name": "upload cluster ca",
"real_userid": {
"domain": "builtin",
"user": "******"
},
"remote": {
"ip": self.master.ip,
"port": 35510
},
"subject": "C=UA, O=MyCompany, CN=clientroot"
}
elif ops == "delete_cluster_CA":
self.x509.upload_root_certs(server=self.master,
root_ca_names=["clientroot"])
ids = self.x509.get_ids_from_ca_names(ca_names=["clientroot"])
self.x509.delete_trusted_CAs(ids=ids)
expectedResults = {
"description": "Delete cluster CA",
"local": {
"ip": self.master.ip,
"port": 8091
},
"name": "delete cluster ca",
"real_userid": {
"domain": "builtin",
"user": "******"
},
"remote": {
"ip": self.master.ip,
"port": 62993
},
"subject": "C=UA, O=MyCompany, CN=clientroot"
}
self.checkConfig(self.eventID, self.master, expectedResults)
self.x509.teardown_certs(servers=self.servers)
def setUp(self):
super(MultipleCANegative, self).setUp()
self.standard = self.input.param("standard", "pkcs8")
self.passphrase_type = self.input.param("passphrase_type", "script")
self.encryption_type = self.input.param("encryption_type", "aes256")
self.x509 = x509main(host=self.master,
standard=self.standard,
encryption_type=self.encryption_type,
passphrase_type=self.passphrase_type)
for server in self.servers:
self.x509.delete_inbox_folder_on_server(server=server)
self.basic_url = "https://" + self.servers[
0].ip + ":18091/pools/default/"
def test_port_security_with_certificates(self):
"""
Scanning the ports to test vulnerabilities with certificates
"""
self.x509 = x509main(host=self.master)
self.x509.generate_multiple_x509_certs(servers=self.servers)
for server in self.servers:
_ = self.x509.upload_root_certs(server)
self.x509.upload_node_certs(servers=self.servers)
self.x509.delete_unused_out_of_the_box_CAs(server=self.master)
self.x509.upload_client_cert_settings(server=self.servers[0])
CbServer.use_https = True
ntonencryptionBase().setup_nton_cluster(servers=self.servers,
clusterEncryptionLevel="strict")
self.test_port_security()
def test_multiple_CAs_offline(self):
"""
1. Init Pre-neo cluster with x509 (single CA)
2. Offline upgrade node one by one
3. Optionally convert pkcs#1 key to encrypted pkcs#8 key in mixed mode cluster
4. After cluster is fully upgraded, rotate certs to use pkcs#8 keys and multiple CAs
"""
self.log.info("------------- test started-------------")
self.generate_x509_certs(self.servers)
self.upload_x509_certs(self.servers)
ntonencryptionBase().setup_nton_cluster(servers=self.servers,
clusterEncryptionLevel="all")
for node in self.servers:
self.log.info(
"-------------Performing upgrade on node {0} to version------------- {1}"
.format(node, self.upgrade_version))
upgrade_threads = self._async_update(
upgrade_version=self.upgrade_version, servers=[node])
for threads in upgrade_threads:
threads.join()
self.log.info("Upgrade finished")
if self.enc_key_mixed_mode:
self.convert_to_pkcs8(node)
self.reload_node_cert_with_plain_password(node)
CbServer.use_https = True
ntonencryptionBase().setup_nton_cluster(
servers=self.servers, clusterEncryptionLevel="strict")
self._reset_original(self.servers)
self.x509_new = x509main(host=self.master,
standard="pkcs8",
encryption_type="aes256",
passphrase_type="script")
self.x509_new.generate_multiple_x509_certs(servers=self.servers)
for server in self.servers:
_ = self.x509_new.upload_root_certs(server)
self.x509_new.upload_node_certs(servers=self.servers)
self.x509_new.delete_unused_out_of_the_box_CAs(server=self.master)
self.x509_new.upload_client_cert_settings(server=self.servers[0])
task = self.cluster.async_rebalance(self.servers[:self.nodes_init],
self.servers[self.nodes_init:], [])
self.wait_for_rebalance_to_complete(task)
self.auth(servers=self.servers)
self.validate_rbac_users_post_upgrade(self.master)
self.validate_user_roles_sdk(self.master)
def test_teardown_with_n2n_encryption(self):
"""
Verify that regenerate, deletion of trusted CAs work
with n2n encryption turned on
"""
CbServer.use_https = True
for level in ["strict", "control", "all"]:
self.x509 = x509main(host=self.master,
standard=self.standard,
encryption_type=self.encryption_type,
passphrase_type=self.passphrase_type,
wildcard_dns=self.wildcard_dns)
self.x509.generate_multiple_x509_certs(
servers=self.servers[:self.nodes_init])
self.x509.upload_root_certs(self.master)
self.x509.upload_node_certs(servers=self.servers[:self.nodes_init])
ntonencryptionBase().setup_nton_cluster(
[self.master], clusterEncryptionLevel=level)
self.x509.teardown_certs(servers=self.servers)
ntonencryptionBase().disable_nton_cluster([self.master])
def test_xdcr_with_security(self):
#Settings
self.settings_values_map = {
"autofailover": ["enable", None],
"n2n": ["enable", "disable"],
"tls": ["all", "control", "strict"]
}
self.apply_settings_before_setup = self._input.param(
"apply_settings_before_setup", False)
self.disable_autofailover = self._input.param("disable_autofailover",
False)
self.enable_n2n = self._input.param("enable_n2n", False)
self.enforce_tls = self._input.param("enforce_tls", None)
self.tls_level = self._input.param("tls_level", "control")
self.enable_autofailover = self._input.param("enable_autofailover",
False)
self.disable_n2n = self._input.param("disable_n2n", None)
self.disable_tls = self._input.param("disable_tls", None)
rebalance_in = self._input.param("rebalance_in", None)
rebalance_out = self._input.param("rebalance_out", None)
swap_rebalance = self._input.param("swap_rebalance", None)
failover = self._input.param("failover", None)
graceful = self._input.param("graceful", None)
pause = self._input.param("pause", None)
reboot = self._input.param("reboot", None)
initial_xdcr = self._input.param("initial_xdcr",
random.choice([True, False]))
random_setting = self._input.param("random_setting", False)
multiple_ca = self._input.param("multiple_ca", None)
use_client_certs = self._input.param("use_client_certs", None)
int_ca_name = self._input.param("int_ca_name", "iclient1_clientroot")
all_node_upload = self._input.param("all_node_upload", False)
rotate_certs = self._input.param("rotate_certs", None)
delete_certs = self._input.param("delete_certs", None)
restart_pkey_nodes = self._input.param("restart_pkey_nodes", None)
if not self.apply_settings_before_setup:
if initial_xdcr:
self.load_and_setup_xdcr()
else:
self.setup_xdcr_and_load()
if self.enforce_tls:
for cluster in self.get_cluster_objects_for_input(
self.enforce_tls):
if self.tls_level == "rotate":
for level in self.settings_values_map["tls"]:
cluster.toggle_security_setting(
[cluster.get_master_node()], "tls", level)
time.sleep(5)
else:
cluster.toggle_security_setting(
[cluster.get_master_node()], "tls", self.tls_level)
#Revert to default (control) tls level
if self.disable_tls:
for cluster in self.get_cluster_objects_for_input(
self.disable_tls):
cluster.toggle_security_setting([cluster.get_master_node()],
"tls")
if self.enable_n2n:
for cluster in self.get_cluster_objects_for_input(self.enable_n2n):
cluster.toggle_security_setting([cluster.get_master_node()],
"n2n", "enable")
if self.disable_n2n:
for cluster in self.get_cluster_objects_for_input(
self.disable_n2n):
cluster.toggle_security_setting([cluster.get_master_node()],
"n2n")
if self.enable_autofailover:
for cluster in self.get_cluster_objects_for_input(
self.enable_autofailover):
cluster.toggle_security_setting([cluster.get_master_node()],
"autofailover", "enable")
if self.disable_autofailover:
for cluster in self.get_cluster_objects_for_input(
self.disable_autofailover):
cluster.toggle_security_setting([cluster.get_master_node()],
"autofailover")
if random_setting:
for cluster in self.get_cluster_objects_for_input(random_setting):
setting = random.choice(list(self.settings_values_map.keys()))
value = random.choice(self.settings_values_map.get(setting))
cluster.toggle_security_setting([cluster.get_master_node()],
setting, value)
if multiple_ca:
for cluster in self.get_cluster_objects_for_input(multiple_ca):
master = cluster.get_master_node()
ntonencryptionBase().disable_nton_cluster([master])
CbServer.x509 = x509main(host=master)
for server in cluster.get_nodes():
CbServer.x509.delete_inbox_folder_on_server(server=server)
CbServer.x509.generate_multiple_x509_certs(
servers=cluster.get_nodes())
if all_node_upload:
for node_num in range(len(cluster.get_nodes())):
CbServer.x509.upload_root_certs(
server=cluster.get_nodes()[node_num],
root_ca_names=[
CbServer.x509.root_ca_names[node_num]
])
else:
for server in cluster.get_nodes():
CbServer.x509.upload_root_certs(server)
CbServer.x509.upload_node_certs(servers=cluster.get_nodes())
if use_client_certs:
CbServer.x509.upload_client_cert_settings(server=master)
client_cert_path, client_key_path = CbServer.x509.get_client_cert(
int_ca_name=int_ca_name)
# Copy the certs onto the test machines
for server in cluster.get_nodes():
shell = RemoteMachineShellConnection(server)
shell.execute_command(
f"mkdir -p {os.path.dirname(client_cert_path)}")
shell.copy_file_local_to_remote(
client_cert_path, client_cert_path)
shell.execute_command(
f"mkdir -p {CbServer.x509.CACERTFILEPATH}all")
shell.copy_file_local_to_remote(
f"{CbServer.x509.CACERTFILEPATH}all/all_ca.pem",
f"{CbServer.x509.CACERTFILEPATH}all/all_ca.pem")
shell.disconnect()
self._client_cert = self._read_from_file(client_cert_path)
self._client_key = self._read_from_file(client_key_path)
self.add_built_in_server_user(node=master)
ntonencryptionBase().setup_nton_cluster(
[master], clusterEncryptionLevel="strict")
if rotate_certs:
for cluster in self.get_cluster_objects_for_input(
rotate_certs):
CbServer.x509.rotate_certs(cluster.get_nodes())
if delete_certs:
for cluster in self.get_cluster_objects_for_input(
delete_certs):
for node in cluster.get_nodes():
CbServer.x509.delete_trusted_CAs(node)
if restart_pkey_nodes:
for cluster in self.get_cluster_objects_for_input(
restart_pkey_nodes):
for node in cluster.get_nodes():
shell = RemoteMachineShellConnection(node)
shell.restart_couchbase()
shell.disconnect()
time.sleep(10)
cluster.failover_and_rebalance_nodes()
cluster.add_back_node("delta")
if self.apply_settings_before_setup:
if initial_xdcr:
self.load_and_setup_xdcr()
else:
self.setup_xdcr_and_load()
if pause:
for cluster in self.get_cluster_objects_for_input(pause):
for remote_cluster_refs in cluster.get_remote_clusters():
remote_cluster_refs.pause_all_replications()
time.sleep(60)
if rebalance_in:
for cluster in self.get_cluster_objects_for_input(rebalance_in):
cluster.rebalance_in()
if failover:
for cluster in self.get_cluster_objects_for_input(failover):
cluster.failover_and_rebalance_nodes(graceful=graceful,
rebalance=True)
if rebalance_out:
for cluster in self.get_cluster_objects_for_input(rebalance_out):
cluster.rebalance_out()
if swap_rebalance:
for cluster in self.get_cluster_objects_for_input(swap_rebalance):
cluster.swap_rebalance()
if pause:
for cluster in self.get_cluster_objects_for_input(pause):
for remote_cluster_refs in cluster.get_remote_clusters():
remote_cluster_refs.resume_all_replications()
if reboot:
for cluster in self.get_cluster_objects_for_input(reboot):
cluster.warmup_node()
time.sleep(60)
self.perform_update_delete()
self.verify_results()
def test_upgrade_with_ldap_root_cert(self):
"""
1. Setup a cluster with x509 certs with n2n encryption enabled.
2. Start an ldap container with a root certificate.
3. Setup an ldap client connection from CB cluster to ldap server.
4. Create an ldap user.
5. Upgrade the CB cluster offline.
6. Add ldap's root cert to cluster's trusted CAs and make ldap
client connection from CB to ldap server using cluster's trusted CAs
7. Validate ldap user authentication works
"""
self.log.info("------------- test started-------------")
self.generate_x509_certs(self.servers)
self.upload_x509_certs(self.servers)
ntonencryptionBase().setup_nton_cluster(servers=self.servers,
clusterEncryptionLevel="all")
# setup and start ldap container
self.log.info("Setting up ldap container")
self.docker = LdapContainer()
shell = RemoteMachineShellConnection(self.master)
self.docker.start_docker(shell)
self.docker.stop_all_containers(shell)
self.docker.remove_all_containers(shell)
self.docker.start_ldap_container(shell)
shell.disconnect()
# Setup ldap config and add an ldap user
self.log.info("Setting up ldap config and creating ldap user")
param = self.get_ldap_params(
hosts='ldap.example.org',
port='636',
encryption='TLS',
bindDN='cn=admin,dc=example,dc=org',
bindPass="******",
serverCertValidation='false',
userDNMapping='{"template":"cn=%u,dc=example,dc=org"}')
self.setup_ldap_config(server=self.master, param=param)
self.add_ldap_user(server=self.master, user_name=self.docker.ldap_user)
for node in self.servers:
self.log.info(
"-------------Performing upgrade on node {0} to version------------- {1}"
.format(node, self.upgrade_version))
upgrade_threads = self._async_update(
upgrade_version=self.upgrade_version, servers=[node])
for threads in upgrade_threads:
threads.join()
self.log.info("Upgrade finished")
# Add multiple CAs with strict level of n2n encryption
CbServer.use_https = True
ntonencryptionBase().setup_nton_cluster(
servers=self.servers, clusterEncryptionLevel="strict")
self._reset_original(self.servers)
self.x509_new = x509main(host=self.master,
standard="pkcs8",
encryption_type="aes256",
passphrase_type="script")
self.x509_new.generate_multiple_x509_certs(servers=self.servers)
for server in self.servers:
_ = self.x509_new.upload_root_certs(server)
self.x509_new.upload_node_certs(servers=self.servers)
self.x509_new.delete_unused_out_of_the_box_CAs(server=self.master)
# Upload ldap's root CA to cluster's trusted CAs
self.log.info("Copying ldap CA to inbox/CA folder")
# TODO: Change the hardcoded path (for now it's okay since the VM on which container is running is fixed )
self.copy_file_from_slave_to_server(
server=self.master,
src=self.docker.ldap_ca,
dst=
"/opt/couchbase/var/lib/couchbase/inbox/CA/ldap_container_ca.crt")
self.log.info("Uploading ldap CA to CB")
self.x509_new.upload_root_certs(self.master)
self.log.info("Changing ldap config to use CB trusted CAs")
param = self.get_ldap_params(
hosts=self.docker.hostname,
port=str(self.docker.ssl_port),
encryption='TLS',
bindDN=self.docker.bindDN,
bindPass=self.docker.bindPass,
serverCertValidation='true',
userDNMapping='{"template":"cn=%u,dc=example,dc=org"}')
self.setup_ldap_config(server=self.master, param=param)
# Validate ldap user post upgrade
ldap_rest = RestConnection(self.master)
ldap_rest.username = self.docker.ldap_user
ldap_rest.password = self.docker.bindPass
status, content, response = self.make_rest_call_with_ldap_user(
ldap_rest=ldap_rest)
if not status:
self.fail(
"Rest call with ldap user credentials failed with content "
"{0}, response{1}".format(content, response))
def test_multiple_CAs_online(self):
"""
1. Init Pre-neo cluster with x509 (single CA)
2. Online upgrade by rebalance-in and rebalance-out
3. Optionally convert pkcs#1 key to encrypted pkcs#8 key in mixed mode cluster
4. After cluster is fully upgraded, rotate certs to use pkcs#8 keys and multiple CAs
"""
self.log.info("------------- test started-------------")
self.generate_x509_certs(self.servers)
self.upload_x509_certs(self.servers)
ntonencryptionBase().setup_nton_cluster(servers=self.servers,
clusterEncryptionLevel="all")
nodes_to_upgrade = self.servers[self.nodes_init:]
nodes_to_upgrade = nodes_to_upgrade[:self.nodes_init]
self.log.info(
"-------------Performing upgrade on node {0} to version------------- {1}"
.format(nodes_to_upgrade, self.upgrade_version))
upgrade_threads = self._async_update(
upgrade_version=self.upgrade_version, servers=nodes_to_upgrade)
for threads in upgrade_threads:
threads.join()
nodes_in_cluster = self.servers[:self.nodes_init]
old_nodes = copy.deepcopy(nodes_in_cluster)
for node in nodes_to_upgrade:
self.log.info("Rebalance-in a Neo node {0}".format(node))
task = self.cluster.async_rebalance(
nodes_in_cluster, [node], [],
services=["kv,n1ql,cbas,eventing,fts,index"])
self.wait_for_rebalance_to_complete(task)
nodes_in_cluster.append(node)
node_out = old_nodes[-1]
self.log.info("Rebalance-out a pre-Neo node {0}".format(node_out))
task = self.cluster.async_rebalance(nodes_in_cluster, [],
[node_out])
self.wait_for_rebalance_to_complete(task)
old_nodes.remove(node_out)
for enode in nodes_in_cluster:
if enode.ip == node_out.ip:
nodes_in_cluster.remove(enode)
self.master = copy.deepcopy(node)
if self.enc_key_mixed_mode:
self.convert_to_pkcs8(node)
self.reload_node_cert_with_plain_password(node)
CbServer.use_https = True
ntonencryptionBase().setup_nton_cluster(
servers=nodes_in_cluster, clusterEncryptionLevel="strict")
self._reset_original(nodes_in_cluster)
self.x509_new = x509main(host=self.master,
standard="pkcs8",
encryption_type="aes256",
passphrase_type="script")
self.x509_new.generate_multiple_x509_certs(servers=nodes_in_cluster)
for server in nodes_in_cluster:
_ = self.x509_new.upload_root_certs(server)
self.x509_new.upload_node_certs(servers=nodes_in_cluster)
self.x509_new.delete_unused_out_of_the_box_CAs(server=self.master)
self.x509_new.upload_client_cert_settings(server=nodes_in_cluster[0])
self.auth(servers=nodes_in_cluster)
self.validate_rbac_users_post_upgrade(self.master)
self.validate_user_roles_sdk(self.master)
def tearDown(self):
self.x509 = x509main(host=self.master)
self.x509.teardown_certs(servers=self.servers)
super(MultipleCANegative, self).tearDown()
