def run(self):
remote_client = RemoteMachineShellConnection(self.server)
output, error = remote_client.execute_command(self.command)
print(self.server.ip)
print("\n".join(output))
print("\n".join(error))
remote_client.disconnect()
def setUp(self):
self.input = TestInputSingleton.input
self.input.test_params.update({"default_bucket": False})
super(volume, self).setUp()
self.bucket_util._expiry_pager(self.cluster, val=5)
self.rest = RestConnection(self.servers[0])
self.available_servers = list()
self.available_servers = self.cluster.servers[self.nodes_init:]
self.nodes_in_cluster = self.cluster.servers[:self.nodes_init]
self.exclude_nodes = [self.cluster.master]
self.skip_check_logs = False
self.iterations = self.input.param("iterations", 1)
self.retry_get_process_num = self.input.param("retry_get_process_num",
400)
self.kv_mem_quota = self.input.param("kv_mem_quota", 22000)
self.index_mem_quota = self.input.param("index_mem_quota", 22700)
self.skip_index_creation_in_setup = self.input.param(
"skip_index_creation_in_setup", False)
self.tombstone_purge_age = self.input.param("tombstone_purge_age", 300)
shell = RemoteMachineShellConnection(self.cluster.master)
shell.enable_diag_eval_on_non_local_hosts()
shell.disconnect()
self.rest.update_tombstone_purge_age_for_removal(
self.tombstone_purge_age)
self.index_setup()
self.fts_setup()
def build_params(self, node):
"""
Builds parameters for node certificate,key upload
"""
script_file = x509main.SCRIPTFILEPATH
shell = RemoteMachineShellConnection(node)
if shell.extract_remote_info().distribution_type == "windows":
script_file = x509main.SCRIPTWINDOWSFILEPATH
shell.disconnect()
params = dict()
if self.encryption_type:
params["privateKeyPassphrase"] = dict()
params["privateKeyPassphrase"]["type"] = self.passphrase_type
if self.passphrase_type == "script":
params["privateKeyPassphrase"]["path"] = self.install_path + \
x509main.SCRIPTSPATH + \
script_file
params["privateKeyPassphrase"]["timeout"] = self.passphrase_load_timeout
params["privateKeyPassphrase"]["trim"] = 'true'
if self.passphrase_script_args:
params["privateKeyPassphrase"]["args"] = self.passphrase_script_args
elif self.passphrase_type == "rest":
params["privateKeyPassphrase"]["url"] = self.passphrase_url
params["privateKeyPassphrase"]["timeout"] = self.passphrase_load_timeout
params["privateKeyPassphrase"]["httpsOpts"] = self.https_opts
else:
params["privateKeyPassphrase"]["type"] = "plain"
params["privateKeyPassphrase"]["password"] = \
self.private_key_passphrase_map[str(node.ip)]
params = json.dumps(params)
return params
def delete_scripts_folder_on_server(self, server=None):
if server is None:
server = self.host
shell = RemoteMachineShellConnection(server)
final_path = self.install_path + x509main.SCRIPTSPATH
shell.execute_command("rm -rf " + final_path)
shell.disconnect()
def create_scripts_folder_on_server(self, server=None):
if server is None:
server = self.host
shell = RemoteMachineShellConnection(server)
final_path = self.install_path + x509main.SCRIPTSPATH
shell.create_directory(final_path)
shell.disconnect()
def get_latest_tombstones_purged_count(self, nodes=None):
"""
grep debug log for the latest tombstones purged count
Return dict with key = node_ip and value = ts purged count
as string
"""
ts_purged_count_dict = dict()
if nodes is None:
nodes = self.cluster_util.get_nodes_in_cluster(self.cluster)
for node in nodes:
shell = RemoteMachineShellConnection(node)
command = "grep 'tombstone_agent:purge:' /opt/couchbase/var/lib/couchbase/logs/debug.log | tail -1"
output, _ = shell.execute_command(command)
try:
if len(output) == 0:
self.log.info(
"Debug.log must have got rotated; trying to find the latest gz file"
)
command = "find /opt/couchbase/var/lib/couchbase/logs -name 'debug.log.*.gz' -print0 " \
"| xargs -r -0 ls -1 -t | tail -1"
output, _ = shell.execute_command(command)
log_file = output[0]
command = "zgrep 'tombstone_agent:purge:' %s | tail -1" % log_file
output, _ = shell.execute_command(command)
self.log.info("On {0} {1}".format(node.ip, output))
shell.disconnect()
purged_count = re.findall("Purged [0-9]+",
output[0])[0].split(" ")[1]
ts_purged_count_dict[node.ip] = purged_count
except Exception as e:
print(e)
ts_purged_count_dict[node.ip] = 0
return ts_purged_count_dict
def split_brain(self, node1, node2):
shell = RemoteMachineShellConnection(node1)
self.log.debug("Blocking traffic from {0} in {1}".format(
node2.ip, node1.ip))
command = "iptables -A INPUT -s {0} -j DROP".format(node2.ip)
shell.execute_command(command)
shell.disconnect()
def _run_curl_command_from_localhost(self, cmd):
"""
Method to run curl cmd get request from localhost by SSHing into self.server
:cmd: curl command (get request)
:returns True, body(content) as a list
:Raises exception if response code not in 200 series
"""
shell = RemoteMachineShellConnection(self.server)
output, error = shell.execute_command(cmd)
shell.disconnect()
if error:
self.log.error(
"Error making Curl request on server {0} {1}".format(
self.server.ip, error))
output_index = 0
for line in output:
if line.startswith("HTTP"):
status_code_line = line.split(" ")
status_code = status_code_line[1]
status_msg = " ".join(status_code_line[2:])
if status_code not in ['200', '201', '202']:
raise Exception("Exception {0} {1}".format(
status_code, status_msg))
elif not line.strip():
content = output[output_index + 1:]
return content
output_index = output_index + 1
def generate_root_certificate(self, root_ca_name, cn_name=None):
root_ca_dir = x509main.CACERTFILEPATH + root_ca_name + "/"
self.create_directory(root_ca_dir)
root_ca_key_path = root_ca_dir + "ca.key"
root_ca_path = root_ca_dir + "ca.pem"
config_path = x509main.ROOT_CA_CONFIG
shell = RemoteMachineShellConnection(self.slave_host)
# create ca.key
output, error = shell.execute_command("openssl genrsa " +
" -out " + root_ca_key_path +
" " + str(self.key_length))
self.log.info('Output message is {0} and error message is {1}'.format(output, error))
if cn_name is None:
cn_name = root_ca_name
# create ca.pem
output, error = shell.execute_command("openssl req -config " + config_path +
" -new -x509 -days 3650" +
" -sha256 -key " + root_ca_key_path +
" -out " + root_ca_path +
" -subj '/C=UA/O=MyCompany/CN=" + cn_name + "'")
self.log.info('Output message is {0} and error message is {1}'.format(output, error))
self.ca_count += 1
self.root_ca_names.append(root_ca_name)
self.manifest[root_ca_name] = dict()
self.manifest[root_ca_name]["path"] = root_ca_dir
self.manifest[root_ca_name]["intermediate"] = dict()
shell.disconnect()
def block_traffic_between_two_nodes(self, node1, node2):
shell = RemoteMachineShellConnection(node1)
self.log.info("Blocking traffic from {0} in {1}".format(
node2.ip, node1.ip))
command = "iptables -A INPUT -s {0} -j DROP".format(node2.ip)
shell.execute_command(command)
shell.disconnect()
def setUp(self):
super(EnforceTls, self).setUp()
self.sample_urls_map = \
{"http://%s:8091/nodes/self": "https://%s:18091/nodes/self",
"http://%s:9102/api/v1/stats": "https://%s:19102/api/v1/stats",
"http://%s:8093/admin/clusters": "https://%s:18093/admin/clusters",
"http://%s:8094/api/cfg": "https://%s:18094/api/cfg",
"http://%s:8096/api/v1/functions": "https://%s:18096/api/v1/functions",
"http://%s:8095/analytics/node/agg/stats/remaining":
"https://%s:18095/analytics/node/agg/stats/remaining",
"http://%s:8097/api/v1/config": "https://%s:18097/api/v1/config"}
self.log.info("Disabling AF on all nodes before beginning the test")
for node in self.cluster.servers:
status = RestConnection(node)\
.update_autofailover_settings(False, 120)
self.assertTrue(status)
self.log.info("Changing security settings to trust all CAs")
trust_all_certs()
self.bucket_util.load_sample_bucket(self.cluster, TravelSample())
shell = RemoteMachineShellConnection(self.cluster.master)
self.curl_path = "/opt/couchbase/bin/curl"
if shell.extract_remote_info().distribution_type == "windows":
self.curl_path = "C:/Program Files/Couchbase/Server/bin/curl"
shell.disconnect()
def setUp(self):
super(RebalanceStartStopTests, self).setUp()
extra_nodes_in = self.input.param("extra_nodes_in", 0)
extra_nodes_out = self.input.param("extra_nodes_out", 0)
self.servs_init = self.servers[:self.nodes_init]
self.servs_in = [self.servers[i + self.nodes_init]
for i in range(self.nodes_in)]
self.servs_out = [self.servers[self.nodes_init - i - 1]
for i in range(self.nodes_out)]
self.extra_servs_in = [self.servers[i + self.nodes_init + self.nodes_in] for i in range(extra_nodes_in)]
self.extra_servs_out = [self.servers[self.nodes_init - i - 1 - self.nodes_out] for i in range(extra_nodes_out)]
self.withMutationOps = self.input.param("withMutationOps", True)
self.sleep_before_rebalance = self.input.param("sleep_before_rebalance", 0)
if self.spec_name is not None:
self.num_items = 20000
self.items = 20000
init_doc_load_spec = \
self.bucket_util.get_crud_template_from_package("initial_load")
# Using the same key as defined in the loading spec
self.gen_update = doc_generator(
init_doc_load_spec["doc_crud"][
MetaCrudParams.DocCrud.COMMON_DOC_KEY],
0, (self.items / 2),
mutation_type="SET")
shell = RemoteMachineShellConnection(self.cluster.master)
shell.enable_diag_eval_on_non_local_hosts()
shell.disconnect()
def create_CA_folder_on_server(self, server=None):
if server is None:
server = self.host
shell = RemoteMachineShellConnection(server)
final_path = self.install_path + x509main.CHAINFILEPATH \
+ "/" + x509main.TRUSTEDCAPATH
shell.create_directory(final_path)
shell.disconnect()
def tearDown(self):
for server in self.cluster.servers[:self.nodes_init]:
shell = RemoteMachineShellConnection(server)
command = "/sbin/iptables -F"
shell.execute_command(command)
shell.disconnect()
self.sleep(10)
super(CollectionsNetworkSplit, self).tearDown()
def set_n2n_encryption_level_on_nodes(self, nodes, level="control"):
self.log.info("Enabling n2n encryption and setting level to "
"{0} on nodes {1}".format(level, nodes))
for node in nodes:
shell_conn = RemoteMachineShellConnection(node)
cb_cli = CbCli(shell_conn, no_ssl_verify=True)
cb_cli.enable_n2n_encryption()
cb_cli.set_n2n_encryption_level(level=level)
shell_conn.disconnect()
def disable_n2n_encryption_cli_on_nodes(self, nodes):
self.set_n2n_encryption_level_on_nodes(nodes=nodes, level="control")
self.log.info("Disabling n2n encryption on nodes {0}".format(nodes))
for node in nodes:
shell_conn = RemoteMachineShellConnection(node)
cb_cli = CbCli(shell_conn, no_ssl_verify=True)
o = cb_cli.disable_n2n_encryption()
self.log.info(o)
shell_conn.disconnect()
def run(self):
remote = RemoteMachineShellConnection(self.server)
server_type = 'membase'
if remote.is_couchbase_installed():
server_type = 'couchbase'
stamp = time.strftime("%d_%m_%Y_%H_%M")
try:
info = remote.extract_remote_info()
if info.type.lower() != 'windows':
core_files = []
print "looking for Erlang/Memcached crashes on {0} " \
"... ".format(
info.ip)
core_files.extend(
remote.file_starts_with(
"/opt/{0}/var/lib/{0}/".format(server_type),
"erl_crash"))
core_files.extend(
remote.file_starts_with(
"/opt/{0}/var/lib/{0}/".format(server_type), "core"))
core_files.extend(remote.file_starts_with("/tmp/", "core"))
core_files.extend(
remote.file_ends_with(
"/opt/{0}/var/lib/{0}/crash".format(server_type),
".dmp"))
if core_files:
print "found dumps on {0}: {1}".format(info.ip, core_files)
command = "mkdir -p /tmp/backup_crash/{0};" \
"mv -f /tmp/core* /tmp/backup_crash/{" \
"0};" \
"mv -f /opt/{1}/var/lib/{" \
"1}/erl_crash.dump* /tmp/backup_crash/{" \
"0}; " \
"mv -f /opt/{1}/var/lib/{1}/crash/*.dmp " \
"" \
"/tmp/backup_crash/{0};". \
format(stamp, server_type)
print "Moved all dumps on {0} to backup folder: " \
"/tmp/backup_crash/{1}".format(
self.server.ip, stamp)
remote.execute_command(command)
output, error = remote.execute_command(
"ls -la /tmp/backup_crash/{0}".format(stamp))
for o in output:
print o
for core_file in core_files:
remote_path, file_name = os.path.dirname(
core_file), os.path.basename(core_file)
if remote.delete_file(remote_path, file_name):
print 'deleted core file : {0}'.format(core_file)
remote.disconnect()
else:
print "dump files not found on {0}".format(info.ip)
if remote:
remote.disconnect()
except Exception as ex:
print ex
def run(self):
remote_client = RemoteMachineShellConnection(self.server)
remote_client.create_file(self.script_name, self.script_content)
output, error = remote_client.execute_command(
"chmod 777 {0} ; {0} ; rm -f {0}".format(self.script_name))
print(self.server.ip)
print("\n".join(output))
print("\n".join(error))
remote_client.disconnect()
def tearDown(self):
for server in self.cluster.servers[:self.nodes_init]:
shell = RemoteMachineShellConnection(server)
command = "/sbin/iptables -F"
shell.execute_command(command)
shell.disconnect()
self.bucket_util._wait_warmup_completed(
self.cluster.servers[:self.nodes_init],
self.bucket_util.buckets[0])
super(CollectionsNetworkSplit, self).tearDown()
def tearDown(self):
# We are not bringing in new nodes, so init nodes should be enough to
# remove iprules
for server in self.known_nodes:
shell = RemoteMachineShellConnection(server)
command = "/sbin/iptables -F"
shell.execute_command(command)
shell.disconnect()
self.sleep(10)
if self.allow_unsafe:
self.wipe_config_on_removed_nodes(self.nodes_failover)
super(CollectionsNetworkSplit, self).tearDown()
def get_node_private_key_passphrase_script(self, server):
"""
Given a server object,
returns the path of the bash script(which prints pkey passphrase for that node) on slave
"""
shell = RemoteMachineShellConnection(server)
if shell.extract_remote_info().distribution_type == "windows":
shell.disconnect()
return self.node_ca_map[str(server.ip)]["path"] + x509main.SCRIPTWINDOWSFILEPATH
else:
shell.disconnect()
return self.node_ca_map[str(server.ip)]["path"] + x509main.SCRIPTFILEPATH
def configure_stats_settings_from_diag_eval(self, key, value):
"""
To change stats config settings through diag/eval
:key: scrape_interval, retention_size, prometheus_metrics_scrape_interval etc
:value: new_value to be set for the above key.
"""
shell = RemoteMachineShellConnection(self.server)
shell.enable_diag_eval_on_non_local_hosts()
shell.disconnect()
key_value = '{%s, %s}' % (key, str(value))
status, content = self.rest.diag_eval(
'ns_config:set_sub(stats_settings, [%s])' % key_value)
if not status:
raise Exception(content)
def custom_induce_failure(self):
"""
Induce failure on nodes
"""
for node in self.server_to_fail:
if self.failover_action == "stop_server":
self.cluster_util.stop_server(node)
elif self.failover_action == "firewall":
self.cluster_util.start_firewall_on_node(node)
elif self.failover_action == "stop_memcached":
self.cluster_util.stop_memcached_on_node(node)
elif self.failover_action == "kill_erlang":
remote = RemoteMachineShellConnection(node)
remote.kill_erlang()
remote.disconnect()
def create_ca_bundle(self):
"""
Creates/updates a pem file with all trusted CAs combined
"""
self.remove_directory(dir_name=x509main.ALL_CAs_PATH)
self.create_directory(dir_name=x509main.ALL_CAs_PATH)
cat_cmd = "cat "
for root_ca, root_ca_manifest in self.manifest.items():
root_ca_dir_path = root_ca_manifest["path"]
root_ca_path = root_ca_dir_path + "ca.pem"
cat_cmd = cat_cmd + root_ca_path + " "
cat_cmd = cat_cmd + "> " + x509main.ALL_CAs_PATH + x509main.ALL_CAs_PEM_NAME
shell = RemoteMachineShellConnection(self.slave_host)
shell.execute_command(cat_cmd)
shell.disconnect()
def kill_indexer(self, server, timeout=10, kill_sleep_time=20):
self.stop_killIndexer = False
counter = 0
while not self.stop_killIndexer:
indexerkill_shell = RemoteMachineShellConnection(server)
counter += 1
if counter > timeout:
break
output, error = indexerkill_shell.kill_indexer()
# output, error = remote_client.execute_command("pkill -f indexer")
self.log.info("Output value is:" + str(output))
self.log.info("Counter value is {0} and max count is {1}".format(
str(counter), str(timeout)))
indexerkill_shell.disconnect()
self.sleep(kill_sleep_time, "Waiting for indexer to warm up")
self.log.info("Kill indexer process for node: {} completed".format(
str(server.ip)))
def test_check_tls_after_restarting_nodes(self):
"""
1. Enforce tls on the cluster
2. Restart couchabse server on all nodes
3. Validate the tls setting has persisted after restart
"""
self.enable_tls_encryption_cli_on_nodes(nodes=[self.cluster.master])
self.log.info("Restarting servers on nodes {0}".format(
self.cluster.servers[:self.nodes_init]))
for node in self.cluster.servers[:self.nodes_init]:
shell = RemoteMachineShellConnection(node)
shell.restart_couchbase()
shell.disconnect()
self.sleep(15, "Wait after restart of servers")
for node in self.cluster.servers[:self.nodes_init]:
level = self.get_encryption_level_on_node(node=node)
if level != "strict":
self.fail("Node {0} expected strict actual {1}".format(
node, level))
def get_ns_config_deleted_keys_count(self, nodes=None):
"""
get a dump of ns_config and grep for "_deleted" to get
deleted keys count
Return dict with key = node_ip and value = deleted key count
as string
"""
deleted_keys_count_dict = dict()
if nodes is None:
nodes = self.cluster_util.get_nodes_in_cluster(self.cluster)
for node in nodes:
shell = RemoteMachineShellConnection(node)
shell.enable_diag_eval_on_non_local_hosts()
command = "curl --silent -u %s:%s http://localhost:8091/diag/eval -d 'ns_config:get()' " \
"| grep '_deleted' | wc -l" % (self.rest.username, self.rest.password)
output, _ = shell.execute_command(command)
shell.disconnect()
deleted_keys_count_dict[node.ip] = output[0].strip('\n')
return deleted_keys_count_dict
def wipe_config_on_removed_nodes(self, removed_nodes=None):
"""
Stop servers on nodes that were failed over and removed, and wipe config dir
"""
if removed_nodes is None:
removed_nodes = self.server_to_fail
for node in removed_nodes:
self.log.info(
"Wiping node config and restarting server on {0}".format(node))
rest = RestConnection(node)
shell = RemoteMachineShellConnection(node)
shell.stop_couchbase()
self.sleep(10)
shell.cleanup_data_config(self.data_path)
shell.start_server()
self.sleep(10)
if not rest.is_ns_server_running():
self.log.error("ns_server {0} is not running.".format(node.ip))
shell.disconnect()
def __init__(self, server):
super(StatsHelper, self).__init__(server)
self.server = server
protocol = "https" if CbServer.use_https else "http"
rest_port = CbServer.ssl_port_map.get(
str(CbServer.port),
CbServer.port) if CbServer.use_https else CbServer.port
fts_port = CbServer.ssl_port_map.get(str(CbServer.fts_port), CbServer.fts_port) \
if CbServer.use_https else CbServer.fts_port
n1ql_port = CbServer.ssl_port_map.get(str(CbServer.n1ql_port), CbServer.n1ql_port) \
if CbServer.use_https else CbServer.n1ql_port
cbas_port = CbServer.ssl_port_map.get(str(CbServer.cbas_port), CbServer.cbas_port) \
if CbServer.use_https else CbServer.cbas_port
eventing_port = CbServer.ssl_port_map.get(str(CbServer.eventing_port), CbServer.eventing_port) \
if CbServer.use_https else CbServer.eventing_port
index_port = CbServer.ssl_port_map.get(str(CbServer.index_port), CbServer.index_port) \
if CbServer.use_https else CbServer.index_port
self.base_url = "{0}://{1}:{2}".format(protocol, self.ip, rest_port)
self.fts_base_url = "{0}://{1}:{2}".format(protocol, self.ip, fts_port)
self.n1ql_base_url = "{0}://{1}:{2}".format(protocol, self.ip,
n1ql_port)
self.cbas_base_url = "{0}://{1}:{2}".format(protocol, self.ip,
cbas_port)
self.eventing_base_url = "{0}://{1}:{2}".format(
protocol, self.ip, eventing_port)
self.index_base_url = "{0}://{1}:{2}".format(protocol, self.ip,
index_port)
self.memcached_ssl_base_url = "http://{0}:{1}".format(self.ip, 11207)
# Prometheus scrapes from KV metrics from this port, and not 11210.
# Look at: /opt/couchbase/var/lib/couchbase/config/prometheus.yaml for ports
self.memcached_base_url = "http://{0}:{1}".format(self.ip, 11280)
self.prometheus_base_url = "http://{0}:{1}".format(self.ip, 9123)
self.rest = RestClientConnection(server)
self.curl_path = "curl"
shell = RemoteMachineShellConnection(self.server)
type = shell.extract_remote_info().distribution_type
if type.lower() == 'windows':
self.path = Windows.COUCHBASE_BIN_PATH
self.curl_path = "%scurl" % self.path
shell.disconnect()
def custom_induce_failure(self, nodes=None):
"""
Induce failure on nodes
"""
if nodes is None:
nodes = self.server_to_fail
for node in nodes:
if self.failover_action == "stop_server":
self.cluster_util.stop_server(self.cluster, node)
elif self.failover_action == "firewall":
self.cluster_util.start_firewall_on_node(self.cluster, node)
elif self.failover_action == "stop_memcached":
self.cluster_util.stop_memcached_on_node(self.cluster, node)
elif self.failover_action == "kill_erlang":
remote = RemoteMachineShellConnection(node)
remote.info = remote.extract_remote_info()
if remote.info.type.lower() == "windows":
remote.kill_erlang(os="windows")
else:
remote.kill_erlang(os="unix")
remote.disconnect()
