def get_lab_setup_files(stream, remote_host=None, release='R6', remote_path=None, local_path=None,
host_type='Standard', ctrlr0_ip=None, username='******', password='******'):
"""
Retrieves necessary setup files from the host specified. If local_path is specified the files in that
directory will be collected else files will be collected from remote_host
Args:
stream(stream): Stream to controller-0, required to put files in correct directories.
remote_host(str): Host to retrieve files from.
release(str): Release to use, if none R5 will be used
remote_path(str): Path to place the files, if none default path will be used
local_path(str): Path to setup files, if none default path to files will be used
host_type(str): Type of host either 'AIO' or 'Standard'
"""
img_dir = "/home/" + username + "/images"
serial.send_bytes(stream, "mkdir " + img_dir)
if local_path:
if not local_path.endswith('/') or not local_path.endswith('\\'):
local_path = local_path + '/'
get_lab_setup_scripts(remote_host, release, remote_path, local_path,
ctrlr0_ip=ctrlr0_ip, username=username, password=password)
else:
get_lab_setup_scripts(remote_host, release, remote_path, local_path,
ctrlr0_ip=ctrlr0_ip, username=username, password=password)
get_licence(remote_host, release, remote_path, local_path, host_type,
ctrlr0_ip=ctrlr0_ip, username=username, password=password)
get_guest_img(stream, remote_host, release, remote_path, local_path,
ctrlr0_ip=ctrlr0_ip, username=username, password=password)
def get_patches(cont0_stream, ctrlr0_ip=None, local_path=None, remote_host=None, release='R6', username='******', password='******'):
"""
Retrieves patches from remote_host or localhost if remote_host is None
"""
patch_dir = "/home/" + username + "/patches"
serial.send_bytes(cont0_stream, "mkdir " + patch_dir)
if local_path is None:
local_path = env.FILEPATH + '{}/patches/'.format(release)
remote_path = '/home/' + username + '/patches/'
LOG.info("Remote host is {}".format(remote_host))
if remote_host is not None:
if release == 'R6':
pass
elif release == 'R5':
#patch_loc = env.Builds.R5['patches']
pass
elif release == 'R4':
patch_loc = env.Builds.R4['patches']
elif release == 'R3':
patch_loc = env.Builds.R3['patches']
else:
patch_loc = env.Builds.R2['patches']
for items in patch_loc:
send_dir(source=local_path, remote_host=ctrlr0_ip, destination=remote_path,
username=username, password=password)
send_dir(source=local_path, remote_host=ctrlr0_ip, destination=remote_path,
username=username, password=password)
else:
LOG.info("Retrieving patches from {}".format(local_path))
if not local_path.endswith('/') or not local_path.endswith('\\'):
local_path = local_path + '/'
send_dir(source=local_path, remote_host=ctrlr0_ip, destination=remote_path,
username=username, password=password)
def logout(stream):
"""
Logs out of controller-0.
Args:
stream(stream): stream to cont0
"""
serial.send_bytes(stream, "exit", expect_prompt=False)
time.sleep(5)
def disable_logout(stream):
"""
Disables automatic logout of users.
Args:
stream(stream): stream to cont0
"""
LOG.info('Disabling automatic logout')
serial.send_bytes(stream, "export TMOUT=0")
def set_dns(stream, dns_ip):
"""
Perform DNS configuration on the system.
"""
LOG.info("Configuring DNS to %s.", dns_ip)
serial.send_bytes(stream,
"source /etc/nova/openrc; system dns-modify "
"nameservers={}".format(dns_ip),
prompt='keystone')
def reboot_host(stream, hostname):
"""
Reboots host specified
Args:
stream():
hostname(str): Host to reboot
"""
LOG.info("Rebooting %s", hostname)
serial.send_bytes(stream, "system host-reboot {}".format(hostname), expect_prompt=False)
serial.expect_bytes(stream, "rebooting", HostTimeout.REBOOT)
def update_platform_cpus(stream, hostname, cpu_num=5):
"""
Update platform CPU allocation.
"""
LOG.info("Allocating %s CPUs for use by the %s platform.", cpu_num,
hostname)
serial.send_bytes(
stream, "\nsource /etc/platform/openrc; system host-cpu-modify "
"{} -f platform -p0 {}".format(hostname,
cpu_num,
prompt='keystone',
timeout=300))
def change_password(stream, username="******", password="******"):
"""
changes the default password on initial login.
Args:
stream(stream): stream to cont0
"""
LOG.info('Changing password to Li69nux*')
serial.send_bytes(stream, username, expect_prompt=False)
serial.expect_bytes(stream, "Password:"******"UNIX password:"******"New password:"******"Retype new")
serial.send_bytes(stream, password)
def setup_networking(stream, release, ctrlr0_ip, hostadapter_ip, password='******'):
"""
Setup initial networking so we can transfer files.
"""
ip = ctrlr0_ip
host_ip = hostadapter_ip
if release == "R2":
interface = "eth0"
else:
interface = "enp0s3"
ret = serial.send_bytes(stream, "/sbin/ip address list", prompt=ctrlr0_ip, fail_ok=True, timeout=10)
if ret != 0:
LOG.info("Setting networking up.")
else:
LOG.info("Skipping networking setup")
return
LOG.info("{} being set up with ip {}".format(interface, ip))
serial.send_bytes(stream, "sudo /sbin/ip addr add {}/24 dev {}".format(ip, interface), expect_prompt=False)
host_helper.check_password(stream, password=password)
time.sleep(2)
serial.send_bytes(stream, "sudo /sbin/ip link set {} up".format(interface), expect_prompt=False)
host_helper.check_password(stream, password=password)
time.sleep(2)
serial.send_bytes(stream, "sudo route add default gw {}".format(host_ip), expect_prompt=False)
host_helper.check_password(stream, password=password)
NETWORKING_TIME = 60
LOG.info("Wait a minute for networking to be established")
time.sleep(NETWORKING_TIME)
def config_controller(stream, config_file=None, password='******'):
"""
Configure controller-0 using optional arguments
"""
args = ''
if config_file:
args += '--config-file ' + config_file + ' '
serial.send_bytes(stream,
"sudo config_controller {}".format(args),
expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream,
"unlock controller to proceed.",
timeout=HostTimeout.LAB_CONFIG)
if ret != 0:
LOG.info("Configuration failed. Exiting installer.")
raise Exception("Configcontroller failed")
def lock_host(stream, hostname):
"""
Locks the specified host.
Args:
stream(stream): Stream to controller-0
hostname(str): Name of host to lock
Steps:
- Check that host is unlocked
- Lock host
"""
LOG.info("Lock %s", hostname)
serial.send_bytes(stream, "system host-list |grep {}".format(hostname), expect_prompt=False)
try:
serial.expect_bytes(stream, "unlocked")
except streamexpect.ExpectTimeout:
LOG.info("Host %s not unlocked", hostname)
return 1
serial.send_bytes(stream, "system host-lock {}".format(hostname), expect_prompt="keystone")
LOG.info("Locking %s", hostname)
def unlock_host(stream, hostname):
"""
Unlocks given host
Args:
stream(stream): Stream to active controller
hostname(str): Name of host to unlock
Steps:
- Check that host is locked
- Unlock host
"""
LOG.info("#### Unlock %s", hostname)
serial.send_bytes(stream, "system host-list | grep {}".format(hostname), expect_prompt=False)
try:
serial.expect_bytes(stream, "locked")
except streamexpect.ExpectTimeout:
LOG.info("Host %s not locked", hostname)
return 1
serial.send_bytes(stream, "system host-unlock {}".format(hostname), expect_prompt=False)
LOG.info("Unlocking %s", hostname)
def login(stream, timeout=600, username="******", password="******"):
"""
Logs into controller-0.
Args:
stream(stream): stream to cont0
timeout(int): Time before login fails in seconds.
"""
serial.send_bytes(stream, "\n", expect_prompt=False)
rc = serial.expect_bytes(stream, "ogin:", fail_ok=True, timeout=timeout)
if rc != 0:
serial.send_bytes(stream, "\n", expect_prompt=False)
if serial.expect_bytes(stream, "~$", timeout=10, fail_ok=True) == -1:
serial.send_bytes(stream, '\n', expect_prompt=False)
serial.expect_bytes(stream, "keystone", timeout=10)
else:
serial.send_bytes(stream, username, expect_prompt=False)
serial.expect_bytes(stream, "assword:")
serial.send_bytes(stream, password)
disable_logout(stream)
def config_controller(stream, default=True, release='R6', config_file=None, backup=None, clone_iso=None,
restore_system=None, restore_images=None, remote_host=None, password='******'):
"""
Configure controller-0 using optional arguments
Args:
stream(stream): stream to controller-0
default(bool): Use default settings
config_file(str): Config file to use.
backup(str):
clone_iso(str):
release(str): Release version
restore_system(str):
restore_images(str):
remote_host(str): Host to retrieve licence from if necessary
Steps:
- Checks for license file
- Runs config_controller with default argument or with config-file if requested
"""
# TODO:Currently only uses config_file and default as optional arguments
args = ''
if config_file:
args += '--config-file ' + config_file
if release != 'R6':
ret = serial.send_bytes(stream, "ls | grep license.lic", prompt='license.lic', fail_ok=True, timeout=10)
if ret != 0:
LOG.info("License file not found. Please retrieve license and press enter to continue.")
input()
if (release == 'R4' or release == 'R3') and not config_file:
LOG.info("Configuration fails for R4/3 when using --default. "
"Please configure manually before continuing the installation")
return 1
LOG.info("Configuring controller-0")
start = time.time()
serial.send_bytes(stream, "sudo config_controller {}".format(args), expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "Configuration was applied", timeout=HostTimeout.LAB_CONFIG)
if ret != 0:
LOG.info("Configuration failed. Exiting installer.")
exit()
kpi.CONFIGTIME = time.time() - start
LOG.info("Configuration time: {} minutes".format(kpi.CONFIGTIME/60))
def install_host(stream, hostname, host_type, host_id):
"""
Initiates install of specified host. Requires controller-0 to be installed already.
Args:
stream(stream): Stream to cont0
hostname(str): Name of host
host_type(str): Type of host being installed e.g. 'storage' or 'compute'
host_id(int): id to identify host
"""
time.sleep(10)
LOG.info("Installing %s with id %s", hostname, host_id)
if host_type is 'controller':
serial.send_bytes(stream,
"system host-update {} personality=controller".format(host_id),
expect_prompt=False)
elif host_type is 'storage':
serial.send_bytes(stream,
"system host-update {} personality=storage".format(host_id),
expect_prompt=False)
else:
serial.send_bytes(stream,
"system host-update {} personality=compute hostname={}".format(host_id,
hostname),
expect_prompt=False)
time.sleep(30)
def get_guest_img(stream, remote_host=None, release='R6', remote_path=None,
local_path=None, ctrlr0_ip=None, username='******', password='******'):
"""
Retrieves necessary setup files from the host specified.
Args:
stream(stream): Stream to controller-0, required to put files in correct directories.
remote_host(str): Host to retrieve files from.
release(str): Release to use, if none R5 will be used
remote_path(str): Path to setup files, if none default path to files will be used
local_path(str): Path on local machine to store files for transfer to vbox
"""
if local_path is None:
local_path = env.FILEPATH + '{}/'.format(release)
file_path = []
if remote_path is None:
if release == 'R6':
file_path = env.Builds.R6['guest']
elif release == 'R5':
file_path = env.Builds.R5['guest']
elif release == 'R4':
file_path = env.Builds.R4['guest']
elif release == 'R3':
file_path = env.Builds.R3['guest']
else:
file_path = env.Builds.R2['guest']
img_dir = "/home/" + username + "/images"
serial.send_bytes(stream, "mkdir " + img_dir)
if release != 'R2':
local_path = local_path + 'tis-centos-guest.img'
else:
local_path = local_path + 'cgcs-guest.img'
if remote_host is not None:
sftp_get(source=file_path, remote_host=remote_host, destination=local_path)
if release != 'R2':
sftp_send(source=local_path, remote_host=ctrlr0_ip, destination="/home/" + username + "/images/tis_centos_guest.img",
username=username, password=password)
else:
sftp_send(source=local_path, remote_host=ctrlr0_ip, destination="/home/" + username + "/images/cgcs-guest.img",
username=username, password=password)
def test_install_nodes(cont0_stream, socks, streams, labname, host_list=None):
"""
Tests node install, requires controller-0 to be installed previously
Args:
cont0_stream:
socks:
streams:
host_list(list): list of host names to install.
Steps:
- Create multiple threads
- Set personalities for nodes
- Wait for nodes to finish installing
13-25 mins
"""
host_id = 2
LOG.info("test_install_nodes")
LOG.info(host_list)
threads = []
new_thread = []
serial.send_bytes(cont0_stream, "source /etc/nova/openrc", prompt='keystone')
for host in host_list:
## Remove the lab name from the host
tis_host = host[len(labname)+1:]
time.sleep(10)
if 'controller' in tis_host:
new_thread.append(threading.InstallThread(cont0_stream, '{} thread'.format(tis_host), tis_host, 'controller',
host_id))
elif 'compute' in tis_host:
new_thread.append(threading.InstallThread(cont0_stream, '{} thread'.format(tis_host), tis_host, 'compute', host_id))
else:
new_thread.append(threading.InstallThread(cont0_stream, '{} thread'.format(tis_host), tis_host, 'storage', host_id))
host_id += 1
for host in new_thread:
host.start()
## WEI Do NOT change this time value
time.sleep(2)
threads.append(host)
for items in threads:
items.join(HostTimeout.HOST_INSTALL)
"""
host_id = 2
LOG.info(host_list)
serial.send_bytes(cont0_stream, "source /etc/nova/openrc", prompt='keystone')
for host in host_list:
if 'controller' in host:
host_helper.install_host(cont0_stream, host, 'controller', host_id)
elif 'compute' in host:
host_helper.install_host(cont0_stream, host, 'compute', host_id)
else:
host_helper.install_host(cont0_stream, host, 'storage', host_id)
host_id += 1
"""
# Now wait for nodes to come up. Look for login.
# Close the socket if we are done
start_time = time.time()
for host in host_list:
try:
serial.expect_bytes(streams[host], "login:"******"{} installation complete".format(host))
# serial.disconnect(socks[host])
except Exception as e:
LOG.info("Connection failed for host {} with {}.".format(host, e))
## Sometimes we get UnicodeDecodeError exception due to the output
## of installation. So try one more time maybe
LOG.info("So ignore the exception and wait for {} login again.".format(host))
if HostTimeout.HOST_INSTALL > (time.time()-start_time):
serial.expect_bytes(streams[host], "login:"******"Node install time: {} minutes".format(kpi.NODEINSTALL/60))
def check_password(stream, password="******"):
ret = serial.expect_bytes(stream, 'assword', fail_ok=True, timeout=5)
if ret == 0:
serial.send_bytes(stream, password, expect_prompt=False)
LOG.info("Length of reboot {} minutes".format(kpi.REBOOTTIME/60))
host_helper.login(stream, username=username, password=password)
## TODO (WEI): Remove it
def enable_lvm(stream, release, password='******'):
"""
Enables LVM backend
Args:
stream: stream to controller-0.
release: Release version installed.
"""
if release != 'R6':
LOG.info("Storage backends configured in config_controller for non R6 releases.")
return
serial.send_bytes(stream, "NODE=controller-0;DEVICE=/dev/sdb;SIZE=10237")
serial.send_bytes(stream, "sudo parted -s $DEVICE mktable gpt", expect_prompt=False)
host_helper.check_password(stream, password=password)
serial.send_bytes(stream, "system host-disk-list 1 | grep /dev/sdb", prompt="10237")
serial.send_bytes(stream, "DISK=$(system host-disk-list $NODE | grep $DEVICE | awk '{print $2}')")
serial.send_bytes(stream, "system host-disk-partition-add $NODE $DISK $SIZE -t lvm_phys_vol")
serial.send_bytes(stream, "system host-lvg-add $NODE cinder-volumes")
serial.send_bytes(stream, "while true; do system host-disk-partition-list $NODE --nowrap | grep $DEVICE | "
"grep Ready; if [ $? -eq 0 ]; then break; fi; sleep 1; done")
serial.send_bytes(stream, "PARTITION=$(system host-disk-partition-list $NODE --disk $DISK --nowrap | grep "
"part1 | awk '{print $2}')")
serial.send_bytes(stream, "system host-pv-add $NODE cinder-volumes $PARTITION")
serial.send_bytes(stream, "system storage-backend-add lvm -s cinder --confirmed")
serial.send_bytes(stream, "while true; do system storage-backend-list | grep lvm | grep configured; if "
"[ $? -eq 0 ]; then break; else sleep 10; fi; done")
def menu_selector(stream, controller_type, securityprofile, release, lowlatency, install_mode='serial'):
"""
Select the correct install option.
Arguments:
* stream(socket) - local domain socket to host
* controller_type(str) - controller_aio, controller_lvm or controller_ceph
* securityprofile(str) - standard or extended
* release(str) - R2, R3, R4, etc.
* lowlatency(bool) - True or False
"""
# R5 only right now.
# Wait for menu to load (add sleep so we can see what is picked)
serial.expect_bytes(stream, "Press")
if release == 'R4':
if controller_type == 'controller_aio':
LOG.info("Selecting AIO controller")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
if lowlatency is True:
LOG.info("Selecting low latency controller")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
if install_mode == 'graphical':
LOG.info("Selecting Graphical menu")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
serial.send_bytes(stream, "\n", expect_prompt=False, send=False)
time.sleep(4)
elif release == 'R3':
if controller_type == "controller_aio":
LOG.info("Selecting AIO controller")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
if install_mode == 'graphical':
LOG.info("Selecting Graphical menu")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
serial.send_bytes(stream, "\n", expect_prompt=False, send=False)
time.sleep(4)
elif release == 'R2':
if controller_type == "controller_aio":
LOG.info("Selecting AIO controller")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
if install_mode == 'graphical':
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
time.sleep(4)
serial.send_bytes(stream, "\n", expect_prompt=False, send=False)
else:
# Pick install type
if controller_type == "controller_aio":
LOG.info("Selecting AIO controller")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
if lowlatency is True:
LOG.info("Selecting low latency controller")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
serial.send_bytes(stream, "\n", expect_prompt=False, send=False)
time.sleep(4)
# Serial or Graphical menu (picking Serial by default)
if install_mode == "graphical":
LOG.info("Selecting Graphical menu")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
else:
LOG.info("Selecting Serial menu")
serial.send_bytes(stream, "\n", expect_prompt=False, send=False)
time.sleep(6)
# Security profile menu
if securityprofile == "extended":
LOG.info("Selecting extended security profile")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
time.sleep(2)
serial.send_bytes(stream, "\n", expect_prompt=False, send=False)
time.sleep(4)
def run_install_scripts(stream, host_list, aio_type=None, storage=False, release='R6', socks=None, streams=None,
labname=None, username='******', password='******', conf_files={}):
"""
Runs lab install.sh iterations. Currently does not support Simplex systems
Args:
stream: Stream to controller-0
host_list: list of hosts, used when running aio scripts to install controller-1 at the appropriate time
release: Release that is installed.
aio_type: Option to run the script for aio setup
storage: Option to run the script for storage setup
streams: Dictionary of streams to nodes
Steps:
- Checks for lab_setup files
- Runs lab_setup iterations
- Unlocks nodes
"""
LOG.info("Starting to run the second round of lab_setup script. ")
if release == 'R6' or release == 'R5' or release == 'R4':
ret = serial.send_bytes(stream, '/bin/ls /home/' + username + '/images/ | grep tis-centos-guest.img',
prompt="tis-centos-guest.img", fail_ok=True,
timeout=10)
else:
ret = serial.send_bytes(stream, '/bin/ls /home/' + username + '/images/ | grep cgcs-guest.img',
prompt="cgcs-guest.img", fail_ok=True,
timeout=10)
if ret != 0:
LOG.info("Guest image not found. Please transfer the file before continuing. "
"Press enter once guest image is obtained.")
input()
time.sleep(5)
serial.send_bytes(stream, "chmod +x *.sh", timeout=20)
ret = serial.send_bytes(stream, '/bin/ls /home/' + username + '/ | grep lab_setup.sh', prompt="lab_setup.sh",
fail_ok=True, timeout=10)
if ret != 0:
LOG.info("Lab_setup.sh not found. Please transfer the "
"required files before continuing. Press enter once files are obtained.")
input()
conf_str = ""
for file in conf_files:
conf_str = conf_str + " -f {}".format(file)
LOG.info("lab_setup.sh will be run with options: {}".format(conf_str))
start = time.time()
if aio_type:
## FOR AIO
serial.send_bytes(stream, "source /etc/nova/openrc", prompt='keystone')
if release != 'R6':
serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False, fail_ok=True)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. "
"Please re-run the iteration before continuing. Press enter to continue.")
input()
LOG.info("Running system compute-config-complete, "
"installation will resume once controller-0 reboots and services are active")
serial.send_bytes(stream, "source /etc/nova/openrc", prompt='keystone')
serial.send_bytes(stream, "system compute-config-complete", expect_prompt=False)
serial.expect_bytes(stream, "login:"******"Installing {} ".format(ctrlr1))
host_helper.install_host(stream, ctrlr1, host_type='controller', host_id=2)
# Now wait for controller-1 to come up. Look for "login" on the serial port.
# Close the socket if we are done
try:
serial.expect_bytes(cont1_stream, "ontroller-1 login:"******"{} installation complete".format(ctrlr1))
except Exception as e:
LOG.info("Connection failed for {} with {}.".format(ctrlr1, e))
## Sometimes we get UnicodeDecodeError exception due to the output
## of installation on the serial port, so ignore the exception and try one more time.
if HostTimeout.HOST_INSTALL > (time.time()-start_time):
LOG.info("Ignore the exception and wait for {} to be installed.".format(ctrlr1))
serial.expect_bytes(cont1_stream, "ontroller-1 login:"******"./lab_setup.sh {}".format(conf_str), expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging."
" Please re-run the iteration before continuing. Press enter to continue.")
input()
## Unlock controller-1
now = time.time()
ret = host_helper.unlock_host(stream, ctrlr1)
if ret == 1:
LOG.info("Cannot unlock controller-1, pausing to allow for debugging. "
"Please unlock before continuing. Press enter to continue.")
input()
retry = 0
while retry < 5:
serial.send_bytes(cont1_stream, '\n', expect_prompt=False)
try:
ret = serial.expect_bytes(cont1_stream, "ontroller-1 login:"******"Unlocking controller-1 timed-out. pausing to allow for debugging. "
"Please unlock before continuing. Press enter to continue.")
input()
break
else:
LOG.info("Unlocking controller-1 time (mins): {}".format((time.time() - now)/60))
if (time.time() - now)/60 < 15.0:
LOG.info("login is found right after host-unlock. Wait and try again.")
now = time.time()
time.sleep(10)
retry += 1
else:
break
except Exception as e:
LOG.info("Unlocking controller-1 failed with {}".format(e))
LOG.info("Pausing to allow for debugging. "
"Please unlock before continuing. Press enter to continue.")
input()
break
serial.disconnect(socks[labname + '-' + ctrlr1])
serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. "
"Please re-run the iteration before continuing. Press enter to continue.")
input()
LOG.info("Completed install successfully.")
else:
## FOR NON-AIO cases
serial.send_bytes(stream, "source /etc/nova/openrc", prompt='keystone')
# TODO (WEI): double check this
# Why only if not R5, we run lab_setup.sh here?
#if release != 'R5':
serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. "
"Please re-run the iteration before continuing. Press enter to continue.")
input()
## TODO:
## 1. Unlock sometimes won't bring a node into enabled state. Need to reboot/reset it.
## 2. When unlocking a node, sometimes the node will reboot again after the "login"
## prompt is seen on the serial port, so seeing "login" doesn't mean the node becomes
## "unlocked/enabled". The workaround can be:
## 1) time.sleep(10) and repeat the wait for "login" logic again.
## 2) After seeing "login" prompt, send to controller-0
## "system host-list | grep node" to look for "enabled".
## Unlock controller-1
now = time.time()
ret = host_helper.unlock_host(stream, "controller-1")
if ret == 1:
LOG.info("Cannot unlock controller-1, pausing to allow for debugging. "
"Please unlock before continuing. Press enter to continue.")
input()
## Wait for controller-1 to come up
retry = 0
ctrlr_1 = labname + "-controller-1"
while retry < 5:
serial.send_bytes(streams[ctrlr_1], '\n', expect_prompt=False)
try:
ret = serial.expect_bytes(streams[ctrlr_1], "ontroller-1 login:"******"Unlocking controller-1 timed-out. pausing to allow for debugging. Press enter to continue.")
input()
break
else:
LOG.info("Unlock controller-1 time (mins): {}".format((time.time() - now)/60))
## Sometimes "login" is found right after "host-unlock" is issued, so we need to wait and try again.
## Also sometimes the node will go thru reboot again after "login" is detected on the serial port.
## As the node unlock time is usually longer than 15 mins, as a temperary workaround
## let's wait at least for 15 mins now.
if (time.time() - now)/60 < 15.0:
LOG.info("Found controller-1 login right away. Need to wait and try again.")
now = time.time()
time.sleep(10)
retry += 1
else:
break
except Exception as e:
LOG.info("Unlock controller-1 failed with {} pausing to allow for debugging. Press enter to continue.".format(e))
input()
break
serial.disconnect(socks[ctrlr_1])
host_list.remove(ctrlr_1)
## TODO: Well, there is a chance that controller-1 is not "unlocked/enabled" yet.
## See comment #2 above.
## If it is a storage lab
if storage:
LOG.info("Re-running lab_setup.sh to configure storage nodes.")
serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. "
"Please re-run the iteration before continuing. Press enter to continue.")
input()
now = time.time()
for host in host_list:
host = host[len(labname)+1:]
if host.startswith('storage'):
LOG.info("Unlock {}".format(host))
ret = host_helper.unlock_host(stream, host)
if ret == 1:
LOG.info("Cannot unlock {}, pausing to allow for debugging. "
"Please unlock before continuing. Press enter to continue.".format(host))
input()
time.sleep(20)
## TODO: Need the retry strategy here too.
for host in host_list:
if 'storage' in host:
serial.send_bytes(streams[host], '\n', expect_prompt=False)
try:
ret = serial.expect_bytes(streams[host], "{} login:"******"Unlocking {} timed-out. Pause for debugging. Press enter to continue.".format(host))
input()
else:
LOG.info("Unlocking {} time (mins): {}".format(host, (time.time() - now)/60))
if (time.time() - now)/60 < 15.0:
LOG.info("Found {} login too soon. Make sure it is up. Press enter to continue.".format(host))
input()
except Exception as e:
LOG.info("Unlock {} failed with {}. Pause for debugging. Press enter to continue.".format(host, e))
input()
serial.disconnect(socks[host])
host_list.remove(host)
LOG.info("Completed storage node unlock")
LOG.info("Re-running lab_setup.sh before unlocking compute nodes.")
serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. "
"Please re-run the iteration before continuing. Press enter to continue.")
input()
# unlock compute nodes
now = time.time()
for host in host_list:
host = host[len(labname)+1:]
ret = host_helper.unlock_host(stream, host)
if ret == 1:
LOG.info("Cannot unlock {}, pausing to allow for debugging. "
"Please unlock before continuing. Press enter to continue.".format(host))
input()
time.sleep(20)
LOG.info("Waiting for {} to unlock.".format(host_list))
## Check unlocking status
failed_nodes = []
for host in host_list:
# TODO Fix it! 'ogin:' is always found immediately after unlock
# WEI: fixed it by retrying
retry = 0
while retry < 5:
serial.send_bytes(streams[host], '\n', expect_prompt=False)
try:
ret = serial.expect_bytes(streams[host], "{} login:"******"Unlock {} timed-out.".format(host))
failed_nodes.append(host)
break
else:
LOG.info("Unlock {} time (mins): {}".format(host, (time.time() - now)/60))
if (time.time() - now)/60 < 15.0:
LOG.info("login is found right after host-unlock. Wait and try again.")
now = time.time()
time.sleep(10)
retry += 1
else:
break
except Exception as e:
LOG.info("Unlock {} failed with {}".format(host, e))
failed_nodes.append(host)
break
serial.disconnect(socks[host])
## Let's reset the VMs that failed to unlock
if failed_nodes:
vboxmanage.vboxmanage_controlvms(failed_nodes, action="reset")
time.sleep(10)
tmp_streams = {}
tmp_socks = {}
LOG.info(failed_nodes)
port = 10001
for host in failed_nodes:
tmp_sock = serial.connect('{}'.format(host), port)
tmp_stream = streamexpect.wrap(tmp_sock, echo=True, close_stream=False)
time.sleep(10)
tmp_socks[host] = tmp_sock
tmp_streams[host] = tmp_stream
port += 1
host_failed = False
for host in failed_nodes:
serial.send_bytes(tmp_streams[host], '\n', expect_prompt=False)
try:
ret = serial.expect_bytes(tmp_streams[host], "{} login:"******"{} timed-out to become unlocked/available after reset.".format(host))
host_failed = True
else:
LOG.info("{} became unlocked/available after reset. time (mins): {}".format(host, (time.time() - now)/60))
except Exception as e:
LOG.info("{} failed to become unlocked/available after reset with {}".format(host, e))
host_failed = True
serial.disconnect(tmp_socks[host])
if host_failed:
LOG.info("Not all the nodes are unlocked successfully. Pausing to allow for debugging. "
"Once they all become unlocked/enabled/available, press enter to continue.")
input()
serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "Done", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. "
"Please re-run the iteration before continuing."
" Press enter to continue.")
input()
LOG.info("Completed lab install.")
kpi.LABTIME = time.time()-start
LOG.info("Lab install time: {}".format(kpi.LABTIME/60))
def lab_setup_controller_0_locked(stream, username='******', password='******', conf_files={}):
"""
Runs initial lab_setup when controller-0 is locked.
This is for R5 only.
Args:
stream: Stream to controller-0
Steps:
- Checks if controller-0 is locked
- Checks for lab_setup files
- Runs first lab_setup iteration
- Unlocks controller-0
"""
time.sleep(10)
serial.send_bytes(stream, "source /etc/nova/openrc", prompt='keystone')
serial.send_bytes(stream, "system host-list", expect_prompt=False)
try:
serial.expect_bytes(stream, "locked")
except streamexpect.ExpectTimeout:
LOG.info("Controller should be locked when configuration is completed.")
return 1
ret = serial.send_bytes(stream, '/bin/ls /home/' + username + '/images/ | grep tis-centos-guest.img',
prompt="tis-centos-guest.img", fail_ok=True, timeout=10)
if ret != 0:
LOG.info("Guest image not found. Please transfer the "
"required files before continuing. Press enter once files are obtained.")
input()
time.sleep(5)
ret = serial.send_bytes(stream, '/bin/ls /home/' + username + '/ | grep lab_setup.sh', prompt="lab_setup.sh",
fail_ok=True, timeout=10)
if ret != 0:
LOG.info("Lab_setup.sh not found. Please transfer the "
"required files before continuing. Press enter once files are obtained.")
input()
time.sleep(5)
conf_str = ""
for file in conf_files:
conf_str = conf_str + " -f {}".format(file)
serial.send_bytes(stream, "sh lab_setup.sh {}".format(conf_str), timeout=HostTimeout.LAB_INSTALL, expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "topping after", timeout=3500, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. "
"Please re-run the iteration before continuing. Press enter to continue.")
input()
start = time.time()
host_helper.unlock_host(stream, 'controller-0')
try:
ret = serial.expect_bytes(stream, 'login:', timeout=HostTimeout.CONTROLLER_UNLOCK)
if ret != 0:
LOG.info("Controller-0 not unlocked,Pausing to allow for debugging. "
"Please re-run the iteration before continuing. Press enter to continue.")
input()
except Exception as e:
LOG.info("Unlock controller-0 failed with {}. ".format(e))
## Sometimes we get UnicodeDecodeError exception due to the output
## on the serial port, so ignore that and try one more time.
LOG.info("Ignore the exception and wait for controller-0 to be unlocked again.")
if HostTimeout.CONTROLLER_UNLOCK > (time.time()-start):
serial.expect_bytes(stream, "login:"******"Controller-0 unlock time: {} minutes".format(end))
LOG.info("Waiting for services to activate.")
time.sleep(60)
