def collect_logs(self):
cmd = "echo " + WRSROOT_PASSWORD + " | sudo -S collect all"
self.sendline(cmd)
while True:
try:
resp = self.expect([
"Are you sure you want to continue"
" connecting (yes/no)?", "password:"******"Compressing Tarball ..: (.*.tar.tgz)", PROMPT
],
timeout=COLLECT_TIMEOUT)
if resp == 0:
self.sendline("yes")
elif resp == 1:
self.sendline(WRSROOT_PASSWORD)
elif resp == 2:
tarball = self.match.group(1)
elif resp == 3:
print("Done!")
break
except pexpect.EOF:
log.exception("Connection closed: "
"Reached EOF in SSH session:"
" {}@{}".format(self.username, self.hostname))
wr_exit()._exit(1, msg)
except pexpect.TIMEOUT as ex:
msg = 'Timeout occurred: Failed to find "{}" in output:' \
'\n{}'.format(expect_pattern, self.before)
log.exception(msg)
wr_exit()._exit(1, msg)
return tarball
def connect(self,
hostname,
username,
password,
prompt=PROMPT,
exit_on_error=1):
"""Establish ssh connection to host."""
try:
log.info("Open SSH connection to {}@{}".format(username, hostname))
self.SSH_OPTS = " -o 'StrictHostKeyChecking=no'" + \
" -o 'UserKnownHostsFile=/dev/null'"
self.PROMPT = prompt
self.force_password = True
self.username = username
self.hostname = hostname
self.login(hostname,
username,
password,
auto_prompt_reset=False,
quiet=False,
login_timeout=SSH_EXPECT_TIMEOUT)
except (pxssh.ExceptionPxssh, pexpect.EOF):
msg = "Failed to login to SSH session: {}@{}".format(
username, hostname)
log.error(msg)
self.close()
if exit_on_error:
wr_exit()._exit(1, msg)
else:
return 1, msg
def rsync(self,
source,
dest_user,
dest_server,
dest,
extra_opts=None,
pre_opts=None,
allow_fail=False):
if extra_opts:
extra_opts_str = " ".join(extra_opts) + " "
else:
extra_opts_str = ""
if not pre_opts:
pre_opts = ""
ssh_opts = '"ssh {}" '.format(" ".join(RSYNC_SSH_OPTIONS))
cmd = "{} rsync -avre {} {} {} ".format(pre_opts, ssh_opts,
extra_opts_str, source)
cmd += "{}@{}:{}".format(dest_user, dest_server, dest)
rc = self.exec_cmd(cmd, RSYNC_TIMEOUT, show_output=False)[0]
if rc != 0:
msg = "Rsync failed"
log.error(msg)
if not allow_fail:
wr_exit()._exit(1, msg)
return rc
def launch_controller1(self):
rc, output = self.ssh_conn.exec_cmd(
"./instances/launch_virtual-controller-1.sh")
if rc is not 0:
msg = " Fail to launch a virtual controller-1."
self.log.error(msg)
wr_exit()._exit(1, msg)
def launch_computes(self):
compute_count = self.get_number_of_computes()
for i in range(0, compute_count):
rc, output = self.ssh_conn.exec_cmd(
"./instances/launch_virtual-compute-" + str(i) + ".sh")
if rc is not 0:
msg = " Fail to launch a virtual compute-" + str(i) + "."
self.log.error(msg)
wr_exit()._exit(1, msg)
def get_ssh_key(self):
rc = self.exec_cmd("test -e " + SSH_KEY_FPATH)[0]
if rc != 0:
# self.sendline(CREATE_PUBLIC_SSH_KEY_CMD)
if self.exec_cmd(
CREATE_PUBLIC_SSH_KEY_CMD.format(SSH_KEY_FPATH))[0] != 0:
msg = "Failed to create public ssh key for user"
log.error(msg)
wr_exit()._exit(1, msg)
ssh_key = self.exec_cmd(
GET_PUBLIC_SSH_KEY_CMD.format(SSH_KEY_FPATH))[1]
return ssh_key
def exec_cmd(self,
cmd,
timeout=SSH_EXPECT_TIMEOUT,
expect_pattern=None,
show_output=True,
fail_ok=False):
output = None
# hide the password if
if cmd.startswith("sshpass"):
log.info(cmd.split(" ", 3)[3])
else:
log.info(cmd)
self.sendline(cmd)
if expect_pattern:
try:
self.expect(expect_pattern, timeout)
except pexpect.EOF:
msg = "Connection closed: Reached EOF in SSH session:" \
" {}@{}".format(self.username, self.hostname)
log.exception(msg)
wr_exit()._exit(1, msg)
except pexpect.TIMEOUT as ex:
msg = 'Timeout occurred: Failed to find "{}" in output:'\
"\n{}".format(expect_pattern, self.before)
log.exception(msg)
if not fail_ok:
wr_exit()._exit(1, msg)
else:
return
output = self.match.group().strip()
log.info("Match: " + output)
self.find_prompt(timeout)
return output
# else:
# output = self.match.group().strip()
# log.info("Match: " + output)
# self.find_prompt(timeout)
# return output
else:
self.find_prompt(timeout)
output = self.get_after()
rc = self.get_rc()
if output and show_output:
log.info("Output:\n" + output)
log.info("Return code: " + rc)
return (int(rc), output)
def validate(self):
file_script = "/folk/{}/{}".format(self.userid, CUMULUS_SETUP_SCRIPT)
file_config = "/folk/{}/{}".format(self.userid,
CUMULUS_SETUP_CFG_FILENAME)
file_cleanup = "/folk/{}/{}".format(self.userid,
CUMULUS_CLEANUP_SCRIPT)
cmd = "test -f {}; test -f {}; test -f {}".format(
file_script, file_config, file_cleanup)
if self.ssh_conn.exec_cmd(cmd)[0] != 0:
msg = "cumulus script files missing; Ensure the following files " \
" exist in /folk/{}: {} {} {}".\
format(file_script, file_config, file_cleanup, self.userid)
self.log.info(msg)
wr_exit()._exit(1, msg)
user_openrc = "/folk/{}/openrc.{}".format(self.userid, self.userid)
if self.ssh_conn.exec_cmd("test -f " + user_openrc)[0] != 0:
msg = "user openrc file missing."
self.log.info(msg)
wr_exit()._exit(1, msg)
# check if floating ips are allocated:
cmd = "source " + user_openrc
cmd += " ; neutron floatingip-list | awk \'/128.224/ { print $5 }\'"
rc, floating_ips = self.ssh_conn.exec_cmd(cmd)
if rc is not 0 or floating_ips is None or \
len(floating_ips.split('\n')) < 3:
msg = "Floating IPs are not allocated to your tenant project:\n " \
" Log into horizon with your keystone user and select your" \
" project. Go to the Project -> Compute -> Access & Security" \
" page. Select the Floating IPs tab. If there aren't three IPs" \
" allocated, use the Allocate IP To Project button to allocate more"
self.log.info(msg)
wr_exit()._exit(1, msg)
self.log.info(floating_ips)
self.floating_ips = floating_ips.split('\n')
self.log.info(floating_ips)
self.cumulus_options = self.parse_cumulus_conf(file_config)
# check if floating ips are specified. in cumulus_setup.conf file
if {"EXTERNALOAMFLOAT", "EXTERNALOAMC0", "EXTERNALOAMC1" } \
not in set(self.cumulus_options) and set(self.floating_ips) \
<= set([self.cumulus_options["EXTERNALOAMFLOAT"], self.cumulus_options["EXTERNALOAMC0"], self.cumulus_options["EXTERNALOAMC1"]]):
msg = "The {} conf file must be updated with allocated floating " \
"ips".format(CUMULUS_SETUP_CFG_FILENAME)
self.log.error(msg)
wr_exit()._exit(1, msg)
for k, v in self.cumulus_options.items():
self.log.info(k + " = " + v + "\n")
def get_after(self):
output = None
after = self.after
if after is pexpect.TIMEOUT:
msg = "Timeout occurred: Failed to find text after executing command"
log.exception(msg)
wr_exit()._exit(1, msg)
lines = after.splitlines()
if len(lines) >= 2:
# Remove date-timestamp and prompt
if re.search(DATE_TIMESTAMP_REGEX, lines[-2]):
output = "\n".join(lines[:-2])
if output is None:
# Remove prompt
output = "\n".join(lines[:-1])
return output
def launch_controller0(self):
# Launch first virtual controller
cumulus_tis_conn = self.ssh_conn
rc, output = cumulus_tis_conn.exec_cmd(
"./instances/launch_virtual-controller-0.sh")
if rc is not 0:
msg = " Fail to launch a virtual controller-0."
self.log.error(msg)
wr_exit()._exit(1, msg)
# wait 5 minutes until the controller-0 boot up and then attempt to set up ssh connection
# using the default 10.10.10.3 oam ip. If success, reset password
floatingip = self.floating_ips[0]
time.sleep(180)
controller0_ssh_conn = SSHClient(log_path=self.output_dir +
"/controller-0.ssh.log")
controller0_ip = self.get_floating_ip("EXTERNALOAMC0")
cmd = "ping -w {} -c 4 {}".format(PING_TIMEOUT, controller0_ip)
ping = 0
while ping < MAX_LOGIN_ATTEMPTS:
self.log.info('Pinging controller-0 with ip address {}'.format(
controller0_ip))
rc, output = cumulus_tis_conn.exec_cmd(cmd)
if rc:
self.log.info("Sleeping for 180 seconds...")
time.sleep(180)
ping += 1
else:
break
if ping == MAX_LOGIN_ATTEMPTS:
msg = 'Waited 1200 seconds and the controller did not respond'
self.log.error(msg)
wr_exit()._exit(1, msg)
ssh_key_fpath = os.path.expanduser(KNOWN_HOSTS_PATH)
if os.path.isfile(ssh_key_fpath):
exec_cmd(
REMOVE_HOSTS_SSH_KEY_CMD.format(ssh_key_fpath,
controller0_ip).split())
cmd = 'ssh wrsroot@' + controller0_ip
controller0_ssh_conn._spawn(cmd)
#controller0_ssh_conn.expect("(yes/no)?")
controller0_ssh_conn.expect(".*\(yes/no\)\? ?$")
controller0_ssh_conn.sendline("yes")
controller0_ssh_conn.expect(PASSWORD_PROMPT)
controller0_ssh_conn.sendline(WRSROOT_DEFAULT_PASSWORD)
controller0_ssh_conn.expect(PASSWORD_PROMPT)
controller0_ssh_conn.sendline(WRSROOT_DEFAULT_PASSWORD)
controller0_ssh_conn.expect(PASSWORD_PROMPT)
controller0_ssh_conn.sendline(WRSROOT_PASSWORD)
controller0_ssh_conn.expect(PASSWORD_PROMPT)
controller0_ssh_conn.sendline(WRSROOT_PASSWORD)
time.sleep(5)
def tis_install(self):
CUMULUS_HOME_DIR = "/folk/" + str(self.userid)
CUMULUS_USERID = self.userid
CUMULUS_SERVER = self.server
CUMULUS_PASSWORD = self.password
self.setSSHConnection()
cumulus_tis_conn = self.ssh_conn
log = self.log
# get tis image
if cumulus_tis_conn.exec_cmd("test -d " +
CUMULUS_TMP_TIS_IMAGE_PATH)[0] == 0:
tis_image_path = CUMULUS_TMP_TIS_IMAGE_PATH + \
"/{}".format(CUMULUS_USERID)
if cumulus_tis_conn.exec_cmd("test -d " + tis_image_path)[0] != 0:
cumulus_tis_conn.sendline("mkdir -p " + tis_image_path)
cumulus_tis_conn.find_prompt()
cumulus_tis_conn.sendline("chmod 755 " + tis_image_path)
cumulus_tis_conn.find_prompt()
else:
tis_image_path = "/folk/{}".format(CUMULUS_USERID)
cmd = "df --output=avail -h {} | sed '1d;s/[^0-9]//g'".format(
tis_image_path)
rc, avail = cumulus_tis_conn.exec_cmd(cmd)
if rc != 0 and avail < BOOT_IMAGE_ISO_SIZE:
msg = "Unable to check available disk space or not sufficient" \
" space in {} for tis image. Currently availabe space is" \
" {}.".format(tis_image_path, avail)
log.exception(msg)
wr_exit()._exit(1, msg)
bld_server_image_path = os.path.join(self.load_path,
BLD_TIS_IMAGE_PATH)
pre_opts = 'sshpass -p "{0}"'.format(CUMULUS_PASSWORD)
self.bld_server_conn.rsync(bld_server_image_path,
CUMULUS_USERID,
CUMULUS_SERVER,
tis_image_path,
pre_opts=pre_opts)
# test if image is downloaded from load server
user_openrc = "/folk/{}/openrc.{}".format(self.userid, self.userid)
cmd = "source {}; test -s {}/{}".format(user_openrc, tis_image_path,
TIS_IMAGE)
if cumulus_tis_conn.exec_cmd(cmd)[0] != 0:
msg = "Failed to download tis image file from load server: {}".\
format(bld_server_image_path)
log.exception(msg)
wr_exit()._exit(1, msg)
# clean up any prior cumumls install
cmd = CUMULUS_HOME_DIR + "/" + CUMULUS_CLEANUP_SCRIPT
if cumulus_tis_conn.exec_cmd(cmd)[0] != 0:
msg = "WARNING: Fail to clean-up previous installation"
log.warning(msg)
# delete if previous tis image if exist in cumulu
cmd = "source {};".format(user_openrc)
cmd += " nova image-list | grep " + CUMULUS_USERID + "-tis | awk \'{print $4}\'"
if cumulus_tis_conn.exec_cmd(cmd)[0] is 0:
# delete image first
cmd = "source {}; nova image-delete {}-tis".format(
user_openrc, CUMULUS_USERID)
if cumulus_tis_conn.exec_cmd(cmd)[0] != 0:
msg = "Failed to delete previous tis image: {}-tis from cumulus".\
format(CUMULUS_USERID)
log.exception(msg)
wr_exit()._exit(1, msg)
# create glance image
log.info(
"Creating glance image using name {}-tis".format(CUMULUS_USERID))
cmd = "source {}; glance image-create --name $USER-tis --container-format bare " \
"--disk-format qcow2 --file {}/{}".format(user_openrc, tis_image_path, TIS_IMAGE)
if cumulus_tis_conn.exec_cmd(cmd, 600)[0] != 0:
msg = "Failed to create tis image from image file: {}".\
format(bld_server_image_path)
log.exception(msg)
wr_exit()._exit(1, msg)
# cumulus_setup: network configuration and create launch scripts for
# virtual controller and compute nodes
cmd = "./" + CUMULUS_SETUP_SCRIPT
rc, output = cumulus_tis_conn.exec_cmd(cmd, 120)
if rc is not 0:
msg = " Fail in {}".format(CUMULUS_SETUP_SCRIPT)
msg += " : " + str(output)
log.error(msg)
wr_exit()._exit(1, msg)
def find_prompt(self, timeout=SSH_EXPECT_TIMEOUT):
matched = self.prompt(timeout)
if not matched:
msg = "Timeout occurred: Failed to find prompt"
log.error(msg)
wr_exit()._exit(1, msg)