class MemcachetestRunner():
def __init__(self,
server,
path="/tmp/",
memcached_ip="localhost",
memcached_port="11211",
num_items=100000,
extra_params=""):
self.server = server
self.shell = RemoteMachineShellConnection(self.server)
self.path = path
self.memcached_ip = memcached_ip
self.memcached_port = memcached_port
self.num_items = num_items
self.extra_params = extra_params
self.log = logger.Logger.get_logger()
def start_memcachetest(self):
# check that memcachetest already installed
exists = self.shell.file_exists('/usr/local/bin/', 'memcachetest')
if not exists:
# try to get from git and install
output, error = self.shell.execute_command_raw(
"cd {0}; git clone git://github.com/membase/memcachetest.git".
format(self.path))
self.shell.log_command_output(output, error)
if "git: command not found" in output[0]:
self.fail("Git should be installed on hosts!")
output, error = self.shell.execute_command_raw(
"cd {0}/memcachetest; ./config/autorun.sh && ./configure && make install"
.format(self.path))
self.shell.log_command_output(output, error)
else:
self.log.info(
"memcachetest already set on {0}:/usr/local/bin/memcachetest".
format(self.server.ip, self.path))
self.stop_memcachetest()
return self.launch_memcachetest()
def launch_memcachetest(self):
exists = self.shell.file_exists('/usr/local/bin/', 'memcachetest')
if not exists:
command = "{0}/memcachetest/memcachetest -h {1}:{2} -i {3} {4}".format(
self.path, self.memcached_ip, self.memcached_port,
self.num_items, self.extra_params)
else:
command = "/usr/local/bin/memcachetest -h {0}:{1} -i {2} {3}".format(
self.memcached_ip, self.memcached_port, self.num_items,
self.extra_params)
output, error = self.shell.execute_command_raw(command)
return self.shell.log_command_output(
output, error, track_words=("downstream timeout", ))
def stop_memcachetest(self):
cmd = "killall memcachetest"
output, error = self.shell.execute_command(cmd)
self.shell.log_command_output(output, error)
self.log.info("memcachetest was stopped on {0}".format(self.server.ip))
def run(self):
remote_client = RemoteMachineShellConnection(self.server)
now = datetime.now()
day = now.day
month = now.month
year = now.year
hour = now.timetuple().tm_hour
minute = now.timetuple().tm_min
file_name = "%s-%s%s%s-%s%s-couch.tar.gz" % (self.server.ip,
month, day, year, hour,
minute)
print("Collecting data files from %s\n" % self.server.ip)
remote_client.extract_remote_info()
data_path = self.__get_data_path(os_type=remote_client.info.type.lower())
output, error = remote_client.execute_command("tar -zcvf {0} '{1}' >/dev/null 2>&1".
format(file_name, data_path))
print("\n".join(output))
print("\n".join(error))
user_path = "/home/"
if self.server.ssh_username == "root":
user_path = "/"
remote_path = "%s%s" % (user_path, self.server.ssh_username)
status = remote_client.file_exists(remote_path, file_name)
if not status:
raise Exception("%s doesn't exists on server" % file_name)
status = remote_client.get_file(remote_path, file_name,
"%s/%s" % (self.path, file_name))
if not status:
raise Exception("Fail to download zipped logs from %s"
% self.server.ip)
remote_client.execute_command("rm -f %s" % os.path.join(remote_path, file_name))
remote_client.disconnect()
def run(self):
file_name = "%s-%s-diag.zip" % (self.server.ip, time_stamp())
if not self.local:
from lib.remote.remote_util import RemoteMachineShellConnection
remote_client = RemoteMachineShellConnection(self.server)
print "Collecting logs from %s\n" % self.server.ip
output, error = remote_client.execute_cbcollect_info(file_name)
print "\n".join(output)
print "\n".join(error)
user_path = "/home/"
if remote_client.info.distribution_type.lower() == "mac":
user_path = "/Users/"
else:
if self.server.ssh_username == "root":
user_path = "/"
remote_path = "%s%s" % (user_path, self.server.ssh_username)
status = remote_client.file_exists(remote_path, file_name)
if not status:
raise Exception("%s doesn't exists on server" % file_name)
status = remote_client.get_file(remote_path, file_name, "%s/%s" % (self.path, file_name))
if status:
print "Downloading zipped logs from %s" % self.server.ip
else:
raise Exception("Fail to download zipped logs from %s" % self.server.ip)
remote_client.execute_command("rm -f %s" % os.path.join(remote_path, file_name))
remote_client.disconnect()
def run(self):
remote_client = RemoteMachineShellConnection(self.server)
now = datetime.now()
day = now.day
month = now.month
year = now.year
hour = now.timetuple().tm_hour
min = now.timetuple().tm_min
file_name = "%s-%s%s%s-%s%s-diag.zip" % (self.server.ip, month, day,
year, hour, min)
print "Collecting logs from %s\n" % self.server.ip
output, error = remote_client.execute_cbcollect_info(file_name)
print "\n".join(output)
print "\n".join(error)
user_path = "/home/"
if self.server.ssh_username == "root":
user_path = "/"
remote_path = "%s%s" % (user_path, self.server.ssh_username)
status = remote_client.file_exists(remote_path, file_name)
if not status:
raise Exception("%s doesn't exists on server" % file_name)
status = remote_client.get_file(remote_path, file_name,
"%s/%s" % (self.path, file_name))
if status:
print "Downloading zipped logs from %s" % self.server.ip
else:
raise Exception("Fail to download zipped logs from %s" %
self.server.ip)
remote_client.execute_command("rm -f %s" %
os.path.join(remote_path, file_name))
remote_client.disconnect()
def run(self):
remote_client = RemoteMachineShellConnection(self.server)
now = datetime.now()
day = now.day
month = now.month
year = now.year
hour = now.timetuple().tm_hour
min = now.timetuple().tm_min
file_name = "%s-%s%s%s-%s%s-diag.zip" % (self.server.ip,
month, day, year, hour, min)
print "Collecting logs from %s\n" % self.server.ip
output, error = remote_client.execute_cbcollect_info(file_name)
print "\n".join(output)
print "\n".join(error)
user_path = "/home/"
if self.server.ssh_username == "root":
user_path = "/"
remote_path = "%s%s" % (user_path, self.server.ssh_username)
status = remote_client.file_exists(remote_path, file_name)
if not status:
raise Exception("%s doesn't exists on server" % file_name)
status = remote_client.get_file(remote_path, file_name,
"%s/%s" % (self.path, file_name))
if status:
print "Downloading zipped logs from %s" % self.server.ip
else:
raise Exception("Fail to download zipped logs from %s"
% self.server.ip)
remote_client.disconnect()
def run(self):
remote_client = RemoteMachineShellConnection(self.server)
now = datetime.now()
day = now.day
month = now.month
year = now.year
hour = now.timetuple().tm_hour
minute = now.timetuple().tm_min
file_name = "%s-%s%s%s-%s%s-couch.tar.gz" % (self.server.ip, month, day, year, hour, minute)
print "Collecting data files from %s\n" % self.server.ip
remote_client.extract_remote_info()
data_path = self.__get_data_path(os_type=remote_client.info.type.lower())
output, error = remote_client.execute_command(
"tar -zcvf {0} '{1}' >/dev/null 2>&1".format(file_name, data_path)
)
print "\n".join(output)
print "\n".join(error)
user_path = "/home/"
if self.server.ssh_username == "root":
user_path = "/"
remote_path = "%s%s" % (user_path, self.server.ssh_username)
status = remote_client.file_exists(remote_path, file_name)
if not status:
raise Exception("%s doesn't exists on server" % file_name)
status = remote_client.get_file(remote_path, file_name, "%s/%s" % (self.path, file_name))
if not status:
raise Exception("Fail to download zipped logs from %s" % self.server.ip)
remote_client.execute_command("rm -f %s" % os.path.join(remote_path, file_name))
remote_client.disconnect()
def start_measure_sched_delays(self):
for server in self.servers:
shell = RemoteMachineShellConnection(server)
exists = shell.file_exists(self.path, 'measure-sched-delays')
if not exists:
shell.copy_file_local_to_remote(
"resources/linux/measure-sched-delays.tar.gz",
"{0}.tar.gz".format(self.path))
output, error = shell.execute_command_raw(
"cd /tmp/; tar -xvzf measure-sched-delays.tar.gz")
shell.log_command_output(output, error)
output, error = shell.execute_command_raw(
"cd {0}; ./configure; make".format(self.path))
shell.log_command_output(output, error)
else:
self.log.info(
"measure-sched-delays already deployed on {0}:{1}".format(
server.ip, self.path))
self.stop_measure_sched_delay()
output, error = shell.execute_command_raw(
"rm -rf {0}/sched-delay*".format(self.path))
shell.log_command_output(output, error)
self.launch_measure_sched_delay(shell,
file="sched-delay-{0}".format(
server.ip))
shell.disconnect()
def run(self):
file_name = "%s-%s-diag.zip" % (self.server.ip, time_stamp())
if not self.local:
from lib.remote.remote_util import RemoteMachineShellConnection
remote_client = RemoteMachineShellConnection(self.server)
print "Collecting logs from %s\n" % self.server.ip
output, error = remote_client.execute_cbcollect_info(file_name)
print "\n".join(error)
user_path = "/home/"
if remote_client.info.distribution_type.lower() == 'mac':
user_path = "/Users/"
else:
if self.server.ssh_username == "root":
user_path = "/"
remote_path = "%s%s" % (user_path, self.server.ssh_username)
status = remote_client.file_exists(remote_path, file_name)
if not status:
raise Exception("%s doesn't exists on server" % file_name)
status = remote_client.get_file(remote_path, file_name,
"%s/%s" % (self.path, file_name))
if status:
print "Downloading zipped logs from %s" % self.server.ip
else:
raise Exception("Fail to download zipped logs from %s" %
self.server.ip)
remote_client.execute_command("rm -f %s" %
os.path.join(remote_path, file_name))
remote_client.disconnect()
def run(self):
file_name = "%s-%s-couch-dbinfo.txt" % (self.server.ip.replace('[', '').replace(']', '').replace(':', '.'),
time_stamp())
if not self.local:
from lib.remote.remote_util import RemoteMachineShellConnection
remote_client = RemoteMachineShellConnection(self.server)
print("Collecting dbinfo from %s\n" % self.server.ip)
output, error = remote_client.execute_couch_dbinfo(file_name)
print("\n".join(output))
print("\n".join(error))
user_path = "/home/"
if remote_client.info.distribution_type.lower() == 'mac':
user_path = "/Users/"
else:
if self.server.ssh_username == "root":
user_path = "/"
remote_path = "%s%s" % (user_path, self.server.ssh_username)
status = remote_client.file_exists(remote_path, file_name)
if not status:
raise Exception("%s doesn't exists on server" % file_name)
status = remote_client.get_file(remote_path, file_name,
"%s/%s" % (self.path, file_name))
if status:
print("Downloading dbinfo logs from %s" % self.server.ip)
else:
raise Exception("Fail to download db logs from %s"
% self.server.ip)
remote_client.execute_command("rm -f %s" % os.path.join(remote_path, file_name))
remote_client.disconnect()
class MemcachetestRunner():
def __init__(self, server, path="/tmp/", memcached_ip="localhost", memcached_port="11211", num_items=100000, extra_params=""):
self.server = server
self.shell = RemoteMachineShellConnection(self.server)
self.path = path
self.memcached_ip = memcached_ip
self.memcached_port = memcached_port
self.num_items = num_items
self.extra_params = extra_params
self.log = logger.Logger.get_logger()
def start_memcachetest(self):
#check that memcachetest already installed
exists = self.shell.file_exists('/usr/local/bin/', 'memcachetest')
if not exists:
#try to get from git and install
output, error = self.shell.execute_command_raw("cd {0}; git clone git://github.com/membase/memcachetest.git".format(self.path))
self.shell.log_command_output(output, error)
output, error = self.shell.execute_command_raw("cd {0}/memcachetest; ./config/autorun.sh && ./configure && make install".format(self.path))
self.shell.log_command_output(output, error)
else:
self.log.info("memcachetest already set on {0}:/usr/local/bin/memcachetest".format(self.server.ip, self.path))
self.stop_memcachetest()
return self.launch_memcachetest()
def launch_memcachetest(self):
command = "{0}/memcachetest/memcachetest -h {1}:{2} -i {3} {4}".format(self.path, self.memcached_ip, self.memcached_port, self.num_items, self.extra_params)
output, error = self.shell.execute_command_raw(command)
status = self.shell.log_command_output(output, error, track_words="downstream timeout")
def stop_memcachetest(self):
cmd = "killall memcachetest"
output, error = self.shell.execute_command(cmd)
self.shell.log_command_output(output, error)
self.log.info("memcachetest was stopped on {0}".format(self.server.ip))
def start_measure_sched_delays(self):
for server in self.servers:
shell = RemoteMachineShellConnection(server)
exists = shell.file_exists(self.path, 'measure-sched-delays')
if not exists:
shell.copy_file_local_to_remote("resources/linux/measure-sched-delays.tar.gz", "{0}.tar.gz".format(self.path))
output, error = shell.execute_command_raw("cd /tmp/; tar -xvzf measure-sched-delays.tar.gz")
shell.log_command_output(output, error)
output, error = shell.execute_command_raw("cd {0}; ./configure; make".format(self.path))
shell.log_command_output(output, error)
else:
self.log.info("measure-sched-delays already deployed on {0}:{1}".format(server.ip, self.path))
self.stop_measure_sched_delay()
output, error = shell.execute_command_raw("rm -rf {0}/sched-delay*".format(self.path))
shell.log_command_output(output, error)
self.launch_measure_sched_delay(shell, file="sched-delay-{0}".format(server.ip))
shell.disconnect()
def _load_snapshot(self, server, bucket, file_base=None, overwrite=True):
"""Load data files from a snapshot"""
dest_data_path = os.path.dirname(server.data_path
or testconstants.COUCHBASE_DATA_PATH)
src_data_path = "{0}-snapshots".format(dest_data_path)
self.log.info(
"server={0}, src_data_path={1}, dest_data_path={2}".format(
server.ip, src_data_path, dest_data_path))
shell = RemoteMachineShellConnection(server)
build_name, short_version, full_version = \
shell.find_build_version("/opt/couchbase/", "VERSION.txt", "cb")
src_file = self._build_tar_name(bucket, full_version, file_base)
if not shell.file_exists(src_data_path, src_file):
self.log.error("file '{0}/{1}' does not exist".format(
src_data_path, src_file))
shell.disconnect()
return False
if not overwrite:
self._save_snapshot(server, bucket, "{0}.tar.gz".format(
time.strftime(PerfDefaults.strftime))) # TODO: filename
rm_cmd = "rm -rf {0}/{1} {0}/{1}-data {0}/_*".format(
dest_data_path, bucket)
self._exec_and_log(shell, rm_cmd)
unzip_cmd = "cd {0}; tar -xvzf {1}/{2}".format(dest_data_path,
src_data_path, src_file)
self._exec_and_log(shell, unzip_cmd)
shell.disconnect()
return True
def _load_snapshot(self, server, bucket, file_base=None, overwrite=True):
"""Load data files from a snapshot"""
dest_data_path = os.path.dirname(server.data_path or
testconstants.COUCHBASE_DATA_PATH)
src_data_path = "{0}-snapshots".format(dest_data_path)
self.log.info("server={0}, src_data_path={1}, dest_data_path={2}"
.format(server.ip, src_data_path, dest_data_path))
shell = RemoteMachineShellConnection(server)
build_name, short_version, full_version = \
shell.find_build_version("/opt/couchbase/", "VERSION.txt", "cb")
src_file = self._build_tar_name(bucket, full_version, file_base)
if not shell.file_exists(src_data_path, src_file):
self.log.error("file '{0}/{1}' does not exist"
.format(src_data_path, src_file))
shell.disconnect()
return False
if not overwrite:
self._save_snapshot(server, bucket,
"{0}.tar.gz".format(
time.strftime(PerfDefaults.strftime))) # TODO: filename
rm_cmd = "rm -rf {0}/{1} {0}/{1}-data {0}/_*".format(dest_data_path,
bucket)
self._exec_and_log(shell, rm_cmd)
unzip_cmd = "cd {0}; tar -xvzf {1}/{2}".format(dest_data_path,
src_data_path, src_file)
self._exec_and_log(shell, unzip_cmd)
shell.disconnect()
return True
