def system_stats(self, nodes, pnames, frequency, verbosity=False):
shells = []
for node in nodes:
try:
bucket = RestConnection(node).get_buckets()[0].name
MemcachedClientHelper.direct_client(node, bucket)
shells.append(RemoteMachineShellConnection(node))
except:
pass
d = {"snapshots": []}
# "pname":"x","pid":"y","snapshots":[{"time":time,"value":value}]
start_time = str(self._task["time"])
while not self._aborted():
time.sleep(frequency)
current_time = time.time()
i = 0
for shell in shells:
node = nodes[i]
unique_id = node.ip + '-' + start_time
for pname in pnames:
obj = RemoteMachineHelper(shell).is_process_running(pname)
if obj and obj.pid:
value = self._extract_proc_info(shell, obj.pid)
value["name"] = pname
value["id"] = obj.pid
value["unique_id"] = unique_id
value["time"] = current_time
value["ip"] = node.ip
d["snapshots"].append(value)
i += 1
self._task["systemstats"] = d["snapshots"]
log.info("Finished system_stats")
def customize_xdcr_settings(self):
"""Set custom XDCR environment variables"""
max_concurrent_reps_per_doc = self.param('max_concurrent_reps_per_doc',
None)
xdcr_doc_batch_size_kb = self.param('xdcr_doc_batch_size_kb', None)
xdcr_checkpoint_interval = self.param('xdcr_checkpoint_interval', None)
xdcr_latency_optimization = self.param('xdcr_latency_optimization',
None)
if max_concurrent_reps_per_doc:
env_var = 'MAX_CONCURRENT_REPS_PER_DOC'
value = max_concurrent_reps_per_doc
elif xdcr_doc_batch_size_kb:
env_var = 'XDCR_DOC_BATCH_SIZE_KB'
value = xdcr_doc_batch_size_kb
elif xdcr_checkpoint_interval:
env_var = 'XDCR_CHECKPOINT_INTERVAL'
value = xdcr_checkpoint_interval
elif xdcr_latency_optimization:
env_var = 'XDCR_LATENCY_OPTIMIZATION'
value = xdcr_latency_optimization
else:
return
self.log.info("changing {0} to {1}".format(env_var, value))
for server in self.input.servers:
rc = RemoteMachineShellConnection(server)
rc.set_environment_variable(env_var, value)
def run(self):
file_name = "%s-%s-couch-dbinfo.txt" % (self.server.ip.replace('[', '').replace(']', '').replace(':', '.'),
time_stamp())
if not self.local:
from lib.remote.remote_util import RemoteMachineShellConnection
remote_client = RemoteMachineShellConnection(self.server)
print("Collecting dbinfo from %s\n" % self.server.ip)
output, error = remote_client.execute_couch_dbinfo(file_name)
print("\n".join(output))
print("\n".join(error))
user_path = "/home/"
if remote_client.info.distribution_type.lower() == 'mac':
user_path = "/Users/"
else:
if self.server.ssh_username == "root":
user_path = "/"
remote_path = "%s%s" % (user_path, self.server.ssh_username)
status = remote_client.file_exists(remote_path, file_name)
if not status:
raise Exception("%s doesn't exists on server" % file_name)
status = remote_client.get_file(remote_path, file_name,
"%s/%s" % (self.path, file_name))
if status:
print("Downloading dbinfo logs from %s" % self.server.ip)
else:
raise Exception("Fail to download db logs from %s"
% self.server.ip)
remote_client.execute_command("rm -f %s" % os.path.join(remote_path, file_name))
remote_client.disconnect()
def load(self, generators_load):
gens_load = []
for generator_load in generators_load:
gens_load.append(copy.deepcopy(generator_load))
items = 0
for gen_load in gens_load:
items += (gen_load.end - gen_load.start)
shell = RemoteMachineShellConnection(self.server)
try:
self.log.info("Delete directory's content %s/data/default/%s ..." %
(self.directory, self.bucket_name))
shell.execute_command('rm -rf %s/data/default/*' % self.directory)
self.log.info("Create directory %s/data/default/%s..." %
(self.directory, self.bucket_name))
shell.execute_command('mkdir -p %s/data/default/%s' %
(self.directory, self.bucket_name))
self.log.info("Load %s documents to %s/data/default/%s..." %
(items, self.directory, self.bucket_name))
for gen_load in gens_load:
for i in range(gen_load.end):
key, value = next(gen_load)
out = shell.execute_command(
"echo '%s' > %s/data/default/%s/%s.json" %
(value, self.directory, self.bucket_name, key))
self.log.info("LOAD IS FINISHED")
finally:
shell.disconnect()
def iostats(self, nodes, frequency, verbosity=False):
shells = []
for node in nodes:
try:
bucket = RestConnection(node).get_buckets()[0].name
MemcachedClientHelper.direct_client(node, bucket)
shells.append(RemoteMachineShellConnection(node))
except:
pass
self._task["iostats"] = []
log.info("Started capturing io stats")
while not self._aborted():
time.sleep(frequency)
log.info("Collecting io_stats")
for shell in shells:
try:
kB_read, kB_wrtn, util, iowait, idle = \
self._extract_io_info(shell)
except (ValueError, TypeError, IndexError):
continue
if kB_read and kB_wrtn:
self._task["iostats"].append({
"time": time.time(),
"ip": shell.ip,
"read": kB_read,
"write": kB_wrtn,
"util": util,
"iowait": iowait,
"idle": idle
})
log.info("Finished capturing io stats")
def convert_to_pkcs8(self, node):
"""
converts a pkcs#1 pkey to pkcs#8 encrypted pkey
directly by executing openssl cmds on VM.
"""
shell = RemoteMachineShellConnection(node)
passw = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(20))
convert_cmd = self.openssl_path + " pkcs8 -in " + self.inbox_folder_path + "pkey.key" +\
" -passout pass:"******" -topk8 -v2 aes256 -out " + \
self.inbox_folder_path + "enckey.key"
output, error = shell.execute_command(convert_cmd)
self.log.info('Output message is {0} and error message is {1}'.format(
output, error))
self.plain_passw_map[node.ip] = passw
remove_cmd = "rm -rf " + self.inbox_folder_path + "pkey.key"
output, error = shell.execute_command(remove_cmd)
self.log.info('Output message is {0} and error message is {1}'.format(
output, error))
mv_command = "mv " + self.inbox_folder_path + "enckey.key " + \
self.inbox_folder_path + "pkey.key"
output, error = shell.execute_command(mv_command)
self.log.info('Output message is {0} and error message is {1}'.format(
output, error))
permissions_cmd = "chmod +777 " + self.inbox_folder_path + "pkey.key"
output, error = shell.execute_command(permissions_cmd)
self.log.info('Output message is {0} and error message is {1}'.format(
output, error))
shell.disconnect()
def run(self):
remote_client = RemoteMachineShellConnection(self.server)
now = datetime.now()
day = now.day
month = now.month
year = now.year
hour = now.timetuple().tm_hour
minute = now.timetuple().tm_min
file_name = "%s-%s%s%s-%s%s-couch.tar.gz" % (self.server.ip,
month, day, year, hour,
minute)
print("Collecting data files from %s\n" % self.server.ip)
remote_client.extract_remote_info()
data_path = self.__get_data_path(os_type=remote_client.info.type.lower())
output, error = remote_client.execute_command("tar -zcvf {0} '{1}' >/dev/null 2>&1".
format(file_name, data_path))
print("\n".join(output))
print("\n".join(error))
user_path = "/home/"
if self.server.ssh_username == "root":
user_path = "/"
remote_path = "%s%s" % (user_path, self.server.ssh_username)
status = remote_client.file_exists(remote_path, file_name)
if not status:
raise Exception("%s doesn't exists on server" % file_name)
status = remote_client.get_file(remote_path, file_name,
"%s/%s" % (self.path, file_name))
if not status:
raise Exception("Fail to download zipped logs from %s"
% self.server.ip)
remote_client.execute_command("rm -f %s" % os.path.join(remote_path, file_name))
remote_client.disconnect()
def check_word_count_eventing_log(self,
function_name,
word,
expected_count,
return_count_only=False):
eventing_nodes = self.get_nodes_from_services_map(
service_type="eventing", get_all_nodes=True)
array_of_counts = []
command = "cat /opt/couchbase/var/lib/couchbase/data/@eventing/" + function_name + "* | grep -a \"" + word + "\" | wc -l"
for eventing_node in eventing_nodes:
shell = RemoteMachineShellConnection(eventing_node)
count, error = shell.execute_non_sudo_command(command)
self.log.info("count : {} and error : {} ".format(count, error))
if isinstance(count, list):
count = int(count[0])
else:
count = int(count)
log.info("Node : {0} , word count on : {1}".format(
eventing_node.ip, count))
array_of_counts.append(count)
count_of_all_words = sum(array_of_counts)
log.info("Total count: {}".format(count_of_all_words))
if return_count_only:
return True, count_of_all_words
if count_of_all_words == expected_count:
return True, count_of_all_words
return False, count_of_all_words
def test_partial_rollback(self):
kv_node = self.get_nodes_from_services_map(service_type="kv", get_all_nodes=True)
log.info("kv nodes:{0}".format(kv_node))
for node in kv_node:
mem_client = MemcachedClientHelper.direct_client(node, self.src_bucket_name)
mem_client.stop_persistence()
body = self.create_save_function_body(self.function_name, self.handler_code,
worker_count=3)
try:
task = self.cluster.async_load_gen_docs(self.master, self.src_bucket_name, self.gens_load,
self.buckets[0].kvs[1], 'create')
except Exception as e:
log.info("error while loading data")
self.deploy_function(body,wait_for_bootstrap=False)
# Kill memcached on Node A
self.log.info("Killing memcached on {0}".format(kv_node[1]))
shell = RemoteMachineShellConnection(kv_node[1])
shell.kill_memcached()
# Start persistence on Node B
self.log.info("Starting persistence on {0}".
format(kv_node[0]))
mem_client = MemcachedClientHelper.direct_client(kv_node[0],
self.src_bucket_name)
mem_client.start_persistence()
# Wait for bootstrap to complete
self.wait_for_bootstrap_to_complete(body['appname'])
stats_src = RestConnection(self.master).get_bucket_stats(bucket=self.src_bucket_name)
log.info(stats_src)
self.verify_eventing_results(self.function_name, stats_src["curr_items"], skip_stats_validation=True)
def test_tls_1_dot_3_ciphers(self):
"""
Verifies Couchbase supports all TLS 1.3 ciphers when TLS minimum version set to 1.3
"""
rest = RestConnection(self.master)
rest.set_min_tls_version(version="tlsv1.3")
for node in self.servers:
self.log.info("Testing node {0}".format(node.ip))
ports_to_scan = self.get_service_ports(node)
ports_to_scan.extend(self.ports_to_scan)
for node_port in ports_to_scan:
self.log.info("Port being tested: {0}".format(node_port))
cmd = self.testssl.TEST_SSL_FILENAME + " -e --warnings off --color 0 {0}:{1}" \
.format(node.ip, node_port)
self.log.info("The command is {0}".format(cmd))
shell = RemoteMachineShellConnection(self.slave_host)
output, error = shell.execute_command(cmd)
shell.disconnect()
output = output.decode().split("\n")
check_next = 0
tls_1_dot_3_ciphers = ["TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256", "TLS_AES_128_CCM_SHA256",
"TLS_AES_128_CCM_8_SHA256"]
for line in output:
if check_next == 1:
if line == '':
check_next = 0
else:
if line.split()[-1] not in tls_1_dot_3_ciphers:
self.fail("Cipher used not under TLS 1.3 supported cipher suites")
elif "--------" in line:
check_next = 1
def test_tls_min_version(self):
"""
Verifies the TLS minimum version of the cluster with the check_version
"""
tls_versions = ["1.3 ", "1.2 ", "1.1 ", "1 "]
for check_version in tls_versions:
self.log.info("Verifying for minimum version = {0}".format(check_version))
rest = RestConnection(self.master)
rest.set_min_tls_version(version="tlsv" + check_version.strip())
for node in self.servers:
self.log.info("Testing node {0}".format(node.ip))
ports_to_scan = self.get_service_ports(node)
ports_to_scan.extend(self.ports_to_scan)
for node_port in ports_to_scan:
self.log.info("Port being tested: {0}".format(node_port))
cmd = self.testssl.TEST_SSL_FILENAME + " -p --warnings off --color 0 {0}:{1}" \
.format(node.ip, node_port)
self.log.info("The command is {0}".format(cmd))
shell = RemoteMachineShellConnection(self.slave_host)
output, error = shell.execute_command(cmd)
shell.disconnect()
output = output.decode().split("\n")
output1 = ''.join(output)
self.assertFalse("error" in output1.lower(), msg=output)
self.assertTrue("tls" in output1.lower(), msg=output)
for line in output:
for version in tls_versions:
if "TLS " + version in line and version >= str(check_version):
self.assertTrue("offered" in line,
msg="TLS {0} is incorrect disabled".format(version))
elif "TLS " + version in line and version < str(check_version):
self.assertTrue("not offered" in line,
msg="TLS {0} is incorrect enabled".format(version))
def create_required_buckets(self):
self.log.info("Get the available memory quota")
bucket_util = bucket_utils(self.master)
self.info = bucket_util.rest.get_nodes_self()
threadhold_memory = 1024
total_memory_in_mb = self.info.memoryFree / 1024 ** 2
total_available_memory_in_mb = total_memory_in_mb
active_service = self.info.services
if "index" in active_service:
total_available_memory_in_mb -= self.info.indexMemoryQuota
if "fts" in active_service:
total_available_memory_in_mb -= self.info.ftsMemoryQuota
if "cbas" in active_service:
total_available_memory_in_mb -= self.info.cbasMemoryQuota
if "eventing" in active_service:
total_available_memory_in_mb -= self.info.eventingMemoryQuota
print(total_memory_in_mb)
available_memory = total_available_memory_in_mb - threadhold_memory
self.rest.set_service_memoryQuota(service='memoryQuota', memoryQuota=available_memory)
self.rest.set_service_memoryQuota(service='cbasMemoryQuota', memoryQuota=available_memory-1024)
self.rest.set_service_memoryQuota(service='indexMemoryQuota', memoryQuota=available_memory-1024)
self.log.info("Create CB buckets")
self.create_bucket(self.master, "GleambookUsers",bucket_ram=available_memory)
shell = RemoteMachineShellConnection(self.master)
command = 'curl -i -u Administrator:password --data \'ns_bucket:update_bucket_props("GleambookUsers", [{extra_config_string, "cursor_dropping_upper_mark=70;cursor_dropping_lower_mark=50"}]).\' http://172.23.104.16:8091/diag/eval'
shell.execute_command(command)
result = RestConnection(self.query_node).query_tool("CREATE PRIMARY INDEX idx_GleambookUsers ON GleambookUsers;")
self.sleep(10, "wait for index creation.")
self.assertTrue(result['status'] == "success")
def test_partial_rollback(self):
kv_node = self.get_nodes_from_services_map(service_type="kv", get_all_nodes=True)
log.info("kv nodes:{0}".format(kv_node))
for node in kv_node:
mem_client = MemcachedClientHelper.direct_client(node, self.src_bucket_name)
mem_client.stop_persistence()
body = self.create_save_function_body(self.function_name, self.handler_code,
worker_count=3)
if self.is_curl:
body['depcfg']['curl'] = []
body['depcfg']['curl'].append({"hostname": self.hostname, "value": "server", "auth_type": self.auth_type,
"username": self.curl_username, "password": self.curl_password,"cookies": self.cookies})
try:
task = self.cluster.async_load_gen_docs(self.master, self.src_bucket_name, self.gens_load,
self.buckets[0].kvs[1], 'create', compression=self.sdk_compression)
except Exception as e:
log.info("error while loading data")
self.deploy_function(body, wait_for_bootstrap=False)
# Kill memcached on Node A
self.log.info("Killing memcached on {0}".format(kv_node[1]))
shell = RemoteMachineShellConnection(kv_node[1])
shell.kill_memcached()
# Start persistence on Node B
self.log.info("Starting persistence on {0}".
format(kv_node[0]))
mem_client = MemcachedClientHelper.direct_client(kv_node[0],
self.src_bucket_name)
mem_client.start_persistence()
self.wait_for_handler_state(body['appname'], "deployed")
stats_src = RestConnection(self.master).get_bucket_stats(bucket=self.src_bucket_name)
log.info(stats_src)
self.verify_eventing_results(self.function_name, stats_src["curr_items"], skip_stats_validation=True)
def remove_backup_repo(self):
remote_client = RemoteMachineShellConnection(self.backup_node)
output, error = remote_client.execute_command("ls " + self.backup_path)
if not error:
command = "rm -rf {0}".format(self.backup_path)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
def run(self):
file_name = "%s-%s-diag.zip" % (self.server.ip, time_stamp())
if not self.local:
from lib.remote.remote_util import RemoteMachineShellConnection
remote_client = RemoteMachineShellConnection(self.server)
print "Collecting logs from %s\n" % self.server.ip
output, error = remote_client.execute_cbcollect_info(file_name)
print "\n".join(error)
user_path = "/home/"
if remote_client.info.distribution_type.lower() == 'mac':
user_path = "/Users/"
else:
if self.server.ssh_username == "root":
user_path = "/"
remote_path = "%s%s" % (user_path, self.server.ssh_username)
status = remote_client.file_exists(remote_path, file_name)
if not status:
raise Exception("%s doesn't exists on server" % file_name)
status = remote_client.get_file(remote_path, file_name,
"%s/%s" % (self.path, file_name))
if status:
print "Downloading zipped logs from %s" % self.server.ip
else:
raise Exception("Fail to download zipped logs from %s" %
self.server.ip)
remote_client.execute_command("rm -f %s" %
os.path.join(remote_path, file_name))
remote_client.disconnect()
def iostats(self, interval=10):
shells = []
for node in self.nodes:
try:
shells.append(RemoteMachineShellConnection(node))
except Exception, error:
log.error(error)
def run(self):
remote_client = RemoteMachineShellConnection(self.server)
now = datetime.now()
day = now.day
month = now.month
year = now.year
hour = now.timetuple().tm_hour
min = now.timetuple().tm_min
file_name = "%s-%s%s%s-%s%s-diag.zip" % (self.server.ip, month, day,
year, hour, min)
print "Collecting logs from %s\n" % self.server.ip
output, error = remote_client.execute_cbcollect_info(file_name)
print "\n".join(output)
print "\n".join(error)
user_path = "/home/"
if self.server.ssh_username == "root":
user_path = "/"
remote_path = "%s%s" % (user_path, self.server.ssh_username)
status = remote_client.file_exists(remote_path, file_name)
if not status:
raise Exception("%s doesn't exists on server" % file_name)
status = remote_client.get_file(remote_path, file_name,
"%s/%s" % (self.path, file_name))
if status:
print "Downloading zipped logs from %s" % self.server.ip
else:
raise Exception("Fail to download zipped logs from %s" %
self.server.ip)
remote_client.execute_command("rm -f %s" %
os.path.join(remote_path, file_name))
remote_client.disconnect()
def copy_file_from_slave_to_server(server, src, dst):
log.info(
"Copying file from slave to server {0}, src{1}, dst{2}".format(
server, src, dst))
shell = RemoteMachineShellConnection(server)
shell.copy_file_local_to_remote(src, dst)
shell.disconnect()
def iostats(self, interval=10):
shells = []
for node in self.nodes:
try:
shells.append(RemoteMachineShellConnection(node))
except Exception as error:
log.error(error)
self._task["iostats"] = []
log.info("started capturing io stats")
while not self._aborted():
time.sleep(interval)
log.info("collecting io stats")
for shell in shells:
try:
kB_read, kB_wrtn, util, iowait, idle = \
self._extract_io_info(shell)
except (ValueError, TypeError, IndexError):
continue
if kB_read and kB_wrtn:
self._task["iostats"].append({
"time": time.time(),
"ip": shell.ip,
"read": kB_read,
"write": kB_wrtn,
"util": util,
"iowait": iowait,
"idle": idle
})
log.info("finished capturing io stats")
def _save_snapshot(self, server, bucket, file_base=None):
"""Save data files to a snapshot"""
src_data_path = os.path.dirname(server.data_path
or testconstants.COUCHBASE_DATA_PATH)
dest_data_path = "{0}-snapshots".format(src_data_path)
self.log.info(
"server={0}, src_data_path={1}, dest_data_path={2}".format(
server.ip, src_data_path, dest_data_path))
shell = RemoteMachineShellConnection(server)
build_name, short_version, full_version = \
shell.find_build_version("/opt/couchbase/", "VERSION.txt", "cb")
dest_file = self._build_tar_name(bucket, full_version, file_base)
self._exec_and_log(shell, "mkdir -p {0}".format(dest_data_path))
# save as gzip file, if file exsits, overwrite
# TODO: multiple buckets
zip_cmd = "cd {0}; tar -cvzf {1}/{2} {3} {3}-data _*"\
.format(src_data_path, dest_data_path, dest_file, bucket)
self._exec_and_log(shell, zip_cmd)
shell.disconnect()
return True
def start_measure_sched_delays(self):
for server in self.servers:
shell = RemoteMachineShellConnection(server)
exists = shell.file_exists(self.path, 'measure-sched-delays')
if not exists:
shell.copy_file_local_to_remote(
"resources/linux/measure-sched-delays.tar.gz",
"{0}.tar.gz".format(self.path))
output, error = shell.execute_command_raw(
"cd /tmp/; tar -xvzf measure-sched-delays.tar.gz")
shell.log_command_output(output, error)
output, error = shell.execute_command_raw(
"cd {0}; ./configure; make".format(self.path))
shell.log_command_output(output, error)
else:
self.log.info(
"measure-sched-delays already deployed on {0}:{1}".format(
server.ip, self.path))
self.stop_measure_sched_delay()
output, error = shell.execute_command_raw(
"rm -rf {0}/sched-delay*".format(self.path))
shell.log_command_output(output, error)
self.launch_measure_sched_delay(shell,
file="sched-delay-{0}".format(
server.ip))
shell.disconnect()
def test_group_autofailover(self):
eject_nodes_structure = self._input.param("eject_nodes", None)
eject_type = self._input.param("eject_type", None)
self.build_cluster()
self.load_data()
idx = self.build_index()
self.wait_for_indexing_complete(item_count=1000)
self.rest.update_autofailover_settings(True,
60,
enableServerGroup=True)
ejected_nodes = self.eject_nodes(
eject_nodes_structure=eject_nodes_structure, eject_type=eject_type)
try:
self.sleep(
120, "Waiting for server group auto failover to be started.")
initial_hits = self.query_node(
index=idx, node=self._cb_cluster.get_fts_nodes()[0])
for zone in self.rest.get_zone_names():
fts_nodes = self.get_zone_healthy_fts_nodes(zone=zone)
for node in fts_nodes:
hits = self.query_node(index=idx, node=node)
self.assertEqual(
initial_hits, hits,
"Difference in search results after server group auto-failover is detected."
)
finally:
for ejected_node in ejected_nodes:
remote = RemoteMachineShellConnection(ejected_node)
remote.start_couchbase()
def system_stats(self, pnames, interval=10):
shells = []
for node in self.nodes:
try:
shells.append(RemoteMachineShellConnection(node))
except Exception as error:
log.error(error)
d = {"snapshots": []}
# "pname":"x","pid":"y","snapshots":[{"time":time,"value":value}]
start_time = str(self._task["time"])
while not self._aborted():
time.sleep(interval)
current_time = time.time()
i = 0
for shell in shells:
node = self.nodes[i]
unique_id = node.ip + '-' + start_time
for pname in pnames:
obj = RemoteMachineHelper(shell).is_process_running(pname)
if obj and obj.pid:
value = self._extract_proc_info(shell, obj.pid)
value["name"] = pname
value["id"] = obj.pid
value["unique_id"] = unique_id
value["time"] = current_time
value["ip"] = node.ip
d["snapshots"].append(value)
i += 1
self._task["systemstats"] = d["snapshots"]
log.info("finished system_stats")
def load_sample_bucket(self, server, bucket_name="travel-sample"):
shell = RemoteMachineShellConnection(server)
shell.execute_command("""curl -v -u Administrator:password \
-X POST http://localhost:8091/sampleBuckets/install \
-d '["{0}"]'""".format(bucket_name))
shell.disconnect()
self.sleep(60)
def load_sample_buckets(self, server, bucketName):
from lib.remote.remote_util import RemoteMachineShellConnection
shell = RemoteMachineShellConnection(server)
shell.execute_command("""curl -v -u Administrator:password \
-X POST http://{0}:8091/sampleBuckets/install \
-d '["{1}"]'""".format(server.ip, bucketName))
shell.disconnect()
self.sleep(20)
def start_atop(self):
"""Start atop collector"""
for node in self.nodes:
shell = RemoteMachineShellConnection(node)
cmd = "killall atop; rm -fr /tmp/*.atop;" + \
"atop -w /tmp/{0}.atop -a 15".format(node.ip) + \
" > /dev/null 2> /dev.null < /dev/null &"
shell.execute_command(cmd)
def set_ep_compaction(self, comp_ratio):
"""Set up ep_engine side compaction ratio"""
for server in self.input.servers:
shell = RemoteMachineShellConnection(server)
cmd = "/opt/couchbase/bin/cbepctl localhost:11210 "\
"set flush_param db_frag_threshold {0}".format(comp_ratio)
self._exec_and_log(shell, cmd)
shell.disconnect()
def __init__(self, server, path="/tmp", memcached_ip="localhost", memcached_port="11211", num_items=100000, extra_params=""):
self.server = server
self.shell = RemoteMachineShellConnection(self.server)
self.path = path
self.memcached_ip = memcached_ip
self.memcached_port = memcached_port
self.num_items = num_items
self.extra_params = extra_params
self.log = logger.Logger.get_logger()
def stop_measure_sched_delay(self):
for server in self.servers:
shell = RemoteMachineShellConnection(server)
cmd = "killall -9 -r .*measure-sched-delays"
output, error = shell.execute_command(cmd)
shell.log_command_output(output, error)
shell.disconnect()
self.log.info("measure-sched-delays was stopped on {0}".format(
server.ip))
def warmup(self, collect_stats=True, flush_os_cache=False):
"""
Restart cluster and wait for it to warm up.
In current version, affect the master node only.
"""
if not self.input.servers:
print "[warmup error] empty server list"
return
if collect_stats:
client_id = self.parami("prefix", 0)
test_params = {
'test_time': time.time(),
'test_name': self.id(),
'json': 0
}
sc = self.start_stats(self.spec_reference + ".warmup",
test_params=test_params,
client_id=client_id)
print "[warmup] preparing to warmup cluster ..."
server = self.input.servers[0]
shell = RemoteMachineShellConnection(server)
start_time = time.time()
print "[warmup] stopping couchbase ... ({0}, {1})"\
.format(server.ip, time.strftime(PerfDefaults.strftime))
shell.stop_couchbase()
print "[warmup] couchbase stopped ({0}, {1})"\
.format(server.ip, time.strftime(PerfDefaults.strftime))
if flush_os_cache:
print "[warmup] flushing os cache ..."
shell.flush_os_caches()
shell.start_couchbase()
print "[warmup] couchbase restarted ({0}, {1})"\
.format(server.ip, time.strftime(PerfDefaults.strftime))
self.wait_until_warmed_up()
print "[warmup] warmup finished"
end_time = time.time()
ops = {
'tot-sets': 0,
'tot-gets': 0,
'tot-items': 0,
'tot-creates': 0,
'tot-misses': 0,
"start-time": start_time,
"end-time": end_time
}
if collect_stats:
self.end_stats(sc, ops, self.spec_reference + ".warmup")
