def get_lab_setup_files(stream, remote_host=None, release='R6', remote_path=None, local_path=None, host_type='Standard', ctrlr0_ip=None, username='******', password='******'): """ Retrieves necessary setup files from the host specified. If local_path is specified the files in that directory will be collected else files will be collected from remote_host Args: stream(stream): Stream to controller-0, required to put files in correct directories. remote_host(str): Host to retrieve files from. release(str): Release to use, if none R5 will be used remote_path(str): Path to place the files, if none default path will be used local_path(str): Path to setup files, if none default path to files will be used host_type(str): Type of host either 'AIO' or 'Standard' """ img_dir = "/home/" + username + "/images" serial.send_bytes(stream, "mkdir " + img_dir) if local_path: if not local_path.endswith('/') or not local_path.endswith('\\'): local_path = local_path + '/' get_lab_setup_scripts(remote_host, release, remote_path, local_path, ctrlr0_ip=ctrlr0_ip, username=username, password=password) else: get_lab_setup_scripts(remote_host, release, remote_path, local_path, ctrlr0_ip=ctrlr0_ip, username=username, password=password) get_licence(remote_host, release, remote_path, local_path, host_type, ctrlr0_ip=ctrlr0_ip, username=username, password=password) get_guest_img(stream, remote_host, release, remote_path, local_path, ctrlr0_ip=ctrlr0_ip, username=username, password=password)
def get_patches(cont0_stream, ctrlr0_ip=None, local_path=None, remote_host=None, release='R6', username='******', password='******'): """ Retrieves patches from remote_host or localhost if remote_host is None """ patch_dir = "/home/" + username + "/patches" serial.send_bytes(cont0_stream, "mkdir " + patch_dir) if local_path is None: local_path = env.FILEPATH + '{}/patches/'.format(release) remote_path = '/home/' + username + '/patches/' LOG.info("Remote host is {}".format(remote_host)) if remote_host is not None: if release == 'R6': pass elif release == 'R5': #patch_loc = env.Builds.R5['patches'] pass elif release == 'R4': patch_loc = env.Builds.R4['patches'] elif release == 'R3': patch_loc = env.Builds.R3['patches'] else: patch_loc = env.Builds.R2['patches'] for items in patch_loc: send_dir(source=local_path, remote_host=ctrlr0_ip, destination=remote_path, username=username, password=password) send_dir(source=local_path, remote_host=ctrlr0_ip, destination=remote_path, username=username, password=password) else: LOG.info("Retrieving patches from {}".format(local_path)) if not local_path.endswith('/') or not local_path.endswith('\\'): local_path = local_path + '/' send_dir(source=local_path, remote_host=ctrlr0_ip, destination=remote_path, username=username, password=password)
def logout(stream): """ Logs out of controller-0. Args: stream(stream): stream to cont0 """ serial.send_bytes(stream, "exit", expect_prompt=False) time.sleep(5)
def disable_logout(stream): """ Disables automatic logout of users. Args: stream(stream): stream to cont0 """ LOG.info('Disabling automatic logout') serial.send_bytes(stream, "export TMOUT=0")
def set_dns(stream, dns_ip): """ Perform DNS configuration on the system. """ LOG.info("Configuring DNS to %s.", dns_ip) serial.send_bytes(stream, "source /etc/nova/openrc; system dns-modify " "nameservers={}".format(dns_ip), prompt='keystone')
def reboot_host(stream, hostname): """ Reboots host specified Args: stream(): hostname(str): Host to reboot """ LOG.info("Rebooting %s", hostname) serial.send_bytes(stream, "system host-reboot {}".format(hostname), expect_prompt=False) serial.expect_bytes(stream, "rebooting", HostTimeout.REBOOT)
def update_platform_cpus(stream, hostname, cpu_num=5): """ Update platform CPU allocation. """ LOG.info("Allocating %s CPUs for use by the %s platform.", cpu_num, hostname) serial.send_bytes( stream, "\nsource /etc/platform/openrc; system host-cpu-modify " "{} -f platform -p0 {}".format(hostname, cpu_num, prompt='keystone', timeout=300))
def change_password(stream, username="******", password="******"): """ changes the default password on initial login. Args: stream(stream): stream to cont0 """ LOG.info('Changing password to Li69nux*') serial.send_bytes(stream, username, expect_prompt=False) serial.expect_bytes(stream, "Password:"******"UNIX password:"******"New password:"******"Retype new") serial.send_bytes(stream, password)
def setup_networking(stream, release, ctrlr0_ip, hostadapter_ip, password='******'): """ Setup initial networking so we can transfer files. """ ip = ctrlr0_ip host_ip = hostadapter_ip if release == "R2": interface = "eth0" else: interface = "enp0s3" ret = serial.send_bytes(stream, "/sbin/ip address list", prompt=ctrlr0_ip, fail_ok=True, timeout=10) if ret != 0: LOG.info("Setting networking up.") else: LOG.info("Skipping networking setup") return LOG.info("{} being set up with ip {}".format(interface, ip)) serial.send_bytes(stream, "sudo /sbin/ip addr add {}/24 dev {}".format(ip, interface), expect_prompt=False) host_helper.check_password(stream, password=password) time.sleep(2) serial.send_bytes(stream, "sudo /sbin/ip link set {} up".format(interface), expect_prompt=False) host_helper.check_password(stream, password=password) time.sleep(2) serial.send_bytes(stream, "sudo route add default gw {}".format(host_ip), expect_prompt=False) host_helper.check_password(stream, password=password) NETWORKING_TIME = 60 LOG.info("Wait a minute for networking to be established") time.sleep(NETWORKING_TIME)
def config_controller(stream, config_file=None, password='******'): """ Configure controller-0 using optional arguments """ args = '' if config_file: args += '--config-file ' + config_file + ' ' serial.send_bytes(stream, "sudo config_controller {}".format(args), expect_prompt=False) host_helper.check_password(stream, password=password) ret = serial.expect_bytes(stream, "unlock controller to proceed.", timeout=HostTimeout.LAB_CONFIG) if ret != 0: LOG.info("Configuration failed. Exiting installer.") raise Exception("Configcontroller failed")
def lock_host(stream, hostname): """ Locks the specified host. Args: stream(stream): Stream to controller-0 hostname(str): Name of host to lock Steps: - Check that host is unlocked - Lock host """ LOG.info("Lock %s", hostname) serial.send_bytes(stream, "system host-list |grep {}".format(hostname), expect_prompt=False) try: serial.expect_bytes(stream, "unlocked") except streamexpect.ExpectTimeout: LOG.info("Host %s not unlocked", hostname) return 1 serial.send_bytes(stream, "system host-lock {}".format(hostname), expect_prompt="keystone") LOG.info("Locking %s", hostname)
def unlock_host(stream, hostname): """ Unlocks given host Args: stream(stream): Stream to active controller hostname(str): Name of host to unlock Steps: - Check that host is locked - Unlock host """ LOG.info("#### Unlock %s", hostname) serial.send_bytes(stream, "system host-list | grep {}".format(hostname), expect_prompt=False) try: serial.expect_bytes(stream, "locked") except streamexpect.ExpectTimeout: LOG.info("Host %s not locked", hostname) return 1 serial.send_bytes(stream, "system host-unlock {}".format(hostname), expect_prompt=False) LOG.info("Unlocking %s", hostname)
def login(stream, timeout=600, username="******", password="******"): """ Logs into controller-0. Args: stream(stream): stream to cont0 timeout(int): Time before login fails in seconds. """ serial.send_bytes(stream, "\n", expect_prompt=False) rc = serial.expect_bytes(stream, "ogin:", fail_ok=True, timeout=timeout) if rc != 0: serial.send_bytes(stream, "\n", expect_prompt=False) if serial.expect_bytes(stream, "~$", timeout=10, fail_ok=True) == -1: serial.send_bytes(stream, '\n', expect_prompt=False) serial.expect_bytes(stream, "keystone", timeout=10) else: serial.send_bytes(stream, username, expect_prompt=False) serial.expect_bytes(stream, "assword:") serial.send_bytes(stream, password) disable_logout(stream)
def config_controller(stream, default=True, release='R6', config_file=None, backup=None, clone_iso=None, restore_system=None, restore_images=None, remote_host=None, password='******'): """ Configure controller-0 using optional arguments Args: stream(stream): stream to controller-0 default(bool): Use default settings config_file(str): Config file to use. backup(str): clone_iso(str): release(str): Release version restore_system(str): restore_images(str): remote_host(str): Host to retrieve licence from if necessary Steps: - Checks for license file - Runs config_controller with default argument or with config-file if requested """ # TODO:Currently only uses config_file and default as optional arguments args = '' if config_file: args += '--config-file ' + config_file if release != 'R6': ret = serial.send_bytes(stream, "ls | grep license.lic", prompt='license.lic', fail_ok=True, timeout=10) if ret != 0: LOG.info("License file not found. Please retrieve license and press enter to continue.") input() if (release == 'R4' or release == 'R3') and not config_file: LOG.info("Configuration fails for R4/3 when using --default. " "Please configure manually before continuing the installation") return 1 LOG.info("Configuring controller-0") start = time.time() serial.send_bytes(stream, "sudo config_controller {}".format(args), expect_prompt=False) host_helper.check_password(stream, password=password) ret = serial.expect_bytes(stream, "Configuration was applied", timeout=HostTimeout.LAB_CONFIG) if ret != 0: LOG.info("Configuration failed. Exiting installer.") exit() kpi.CONFIGTIME = time.time() - start LOG.info("Configuration time: {} minutes".format(kpi.CONFIGTIME/60))
def install_host(stream, hostname, host_type, host_id): """ Initiates install of specified host. Requires controller-0 to be installed already. Args: stream(stream): Stream to cont0 hostname(str): Name of host host_type(str): Type of host being installed e.g. 'storage' or 'compute' host_id(int): id to identify host """ time.sleep(10) LOG.info("Installing %s with id %s", hostname, host_id) if host_type is 'controller': serial.send_bytes(stream, "system host-update {} personality=controller".format(host_id), expect_prompt=False) elif host_type is 'storage': serial.send_bytes(stream, "system host-update {} personality=storage".format(host_id), expect_prompt=False) else: serial.send_bytes(stream, "system host-update {} personality=compute hostname={}".format(host_id, hostname), expect_prompt=False) time.sleep(30)
def get_guest_img(stream, remote_host=None, release='R6', remote_path=None, local_path=None, ctrlr0_ip=None, username='******', password='******'): """ Retrieves necessary setup files from the host specified. Args: stream(stream): Stream to controller-0, required to put files in correct directories. remote_host(str): Host to retrieve files from. release(str): Release to use, if none R5 will be used remote_path(str): Path to setup files, if none default path to files will be used local_path(str): Path on local machine to store files for transfer to vbox """ if local_path is None: local_path = env.FILEPATH + '{}/'.format(release) file_path = [] if remote_path is None: if release == 'R6': file_path = env.Builds.R6['guest'] elif release == 'R5': file_path = env.Builds.R5['guest'] elif release == 'R4': file_path = env.Builds.R4['guest'] elif release == 'R3': file_path = env.Builds.R3['guest'] else: file_path = env.Builds.R2['guest'] img_dir = "/home/" + username + "/images" serial.send_bytes(stream, "mkdir " + img_dir) if release != 'R2': local_path = local_path + 'tis-centos-guest.img' else: local_path = local_path + 'cgcs-guest.img' if remote_host is not None: sftp_get(source=file_path, remote_host=remote_host, destination=local_path) if release != 'R2': sftp_send(source=local_path, remote_host=ctrlr0_ip, destination="/home/" + username + "/images/tis_centos_guest.img", username=username, password=password) else: sftp_send(source=local_path, remote_host=ctrlr0_ip, destination="/home/" + username + "/images/cgcs-guest.img", username=username, password=password)
def test_install_nodes(cont0_stream, socks, streams, labname, host_list=None): """ Tests node install, requires controller-0 to be installed previously Args: cont0_stream: socks: streams: host_list(list): list of host names to install. Steps: - Create multiple threads - Set personalities for nodes - Wait for nodes to finish installing 13-25 mins """ host_id = 2 LOG.info("test_install_nodes") LOG.info(host_list) threads = [] new_thread = [] serial.send_bytes(cont0_stream, "source /etc/nova/openrc", prompt='keystone') for host in host_list: ## Remove the lab name from the host tis_host = host[len(labname)+1:] time.sleep(10) if 'controller' in tis_host: new_thread.append(threading.InstallThread(cont0_stream, '{} thread'.format(tis_host), tis_host, 'controller', host_id)) elif 'compute' in tis_host: new_thread.append(threading.InstallThread(cont0_stream, '{} thread'.format(tis_host), tis_host, 'compute', host_id)) else: new_thread.append(threading.InstallThread(cont0_stream, '{} thread'.format(tis_host), tis_host, 'storage', host_id)) host_id += 1 for host in new_thread: host.start() ## WEI Do NOT change this time value time.sleep(2) threads.append(host) for items in threads: items.join(HostTimeout.HOST_INSTALL) """ host_id = 2 LOG.info(host_list) serial.send_bytes(cont0_stream, "source /etc/nova/openrc", prompt='keystone') for host in host_list: if 'controller' in host: host_helper.install_host(cont0_stream, host, 'controller', host_id) elif 'compute' in host: host_helper.install_host(cont0_stream, host, 'compute', host_id) else: host_helper.install_host(cont0_stream, host, 'storage', host_id) host_id += 1 """ # Now wait for nodes to come up. Look for login. # Close the socket if we are done start_time = time.time() for host in host_list: try: serial.expect_bytes(streams[host], "login:"******"{} installation complete".format(host)) # serial.disconnect(socks[host]) except Exception as e: LOG.info("Connection failed for host {} with {}.".format(host, e)) ## Sometimes we get UnicodeDecodeError exception due to the output ## of installation. So try one more time maybe LOG.info("So ignore the exception and wait for {} login again.".format(host)) if HostTimeout.HOST_INSTALL > (time.time()-start_time): serial.expect_bytes(streams[host], "login:"******"Node install time: {} minutes".format(kpi.NODEINSTALL/60))
def check_password(stream, password="******"): ret = serial.expect_bytes(stream, 'assword', fail_ok=True, timeout=5) if ret == 0: serial.send_bytes(stream, password, expect_prompt=False)
LOG.info("Length of reboot {} minutes".format(kpi.REBOOTTIME/60)) host_helper.login(stream, username=username, password=password) ## TODO (WEI): Remove it def enable_lvm(stream, release, password='******'): """ Enables LVM backend Args: stream: stream to controller-0. release: Release version installed. """ if release != 'R6': LOG.info("Storage backends configured in config_controller for non R6 releases.") return serial.send_bytes(stream, "NODE=controller-0;DEVICE=/dev/sdb;SIZE=10237") serial.send_bytes(stream, "sudo parted -s $DEVICE mktable gpt", expect_prompt=False) host_helper.check_password(stream, password=password) serial.send_bytes(stream, "system host-disk-list 1 | grep /dev/sdb", prompt="10237") serial.send_bytes(stream, "DISK=$(system host-disk-list $NODE | grep $DEVICE | awk '{print $2}')") serial.send_bytes(stream, "system host-disk-partition-add $NODE $DISK $SIZE -t lvm_phys_vol") serial.send_bytes(stream, "system host-lvg-add $NODE cinder-volumes") serial.send_bytes(stream, "while true; do system host-disk-partition-list $NODE --nowrap | grep $DEVICE | " "grep Ready; if [ $? -eq 0 ]; then break; fi; sleep 1; done") serial.send_bytes(stream, "PARTITION=$(system host-disk-partition-list $NODE --disk $DISK --nowrap | grep " "part1 | awk '{print $2}')") serial.send_bytes(stream, "system host-pv-add $NODE cinder-volumes $PARTITION") serial.send_bytes(stream, "system storage-backend-add lvm -s cinder --confirmed") serial.send_bytes(stream, "while true; do system storage-backend-list | grep lvm | grep configured; if " "[ $? -eq 0 ]; then break; else sleep 10; fi; done")
def menu_selector(stream, controller_type, securityprofile, release, lowlatency, install_mode='serial'): """ Select the correct install option. Arguments: * stream(socket) - local domain socket to host * controller_type(str) - controller_aio, controller_lvm or controller_ceph * securityprofile(str) - standard or extended * release(str) - R2, R3, R4, etc. * lowlatency(bool) - True or False """ # R5 only right now. # Wait for menu to load (add sleep so we can see what is picked) serial.expect_bytes(stream, "Press") if release == 'R4': if controller_type == 'controller_aio': LOG.info("Selecting AIO controller") serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) if lowlatency is True: LOG.info("Selecting low latency controller") serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) if install_mode == 'graphical': LOG.info("Selecting Graphical menu") serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) serial.send_bytes(stream, "\n", expect_prompt=False, send=False) time.sleep(4) elif release == 'R3': if controller_type == "controller_aio": LOG.info("Selecting AIO controller") serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) if install_mode == 'graphical': LOG.info("Selecting Graphical menu") serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) serial.send_bytes(stream, "\n", expect_prompt=False, send=False) time.sleep(4) elif release == 'R2': if controller_type == "controller_aio": LOG.info("Selecting AIO controller") serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) if install_mode == 'graphical': serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) time.sleep(4) serial.send_bytes(stream, "\n", expect_prompt=False, send=False) else: # Pick install type if controller_type == "controller_aio": LOG.info("Selecting AIO controller") serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) if lowlatency is True: LOG.info("Selecting low latency controller") serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) serial.send_bytes(stream, "\n", expect_prompt=False, send=False) time.sleep(4) # Serial or Graphical menu (picking Serial by default) if install_mode == "graphical": LOG.info("Selecting Graphical menu") serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) else: LOG.info("Selecting Serial menu") serial.send_bytes(stream, "\n", expect_prompt=False, send=False) time.sleep(6) # Security profile menu if securityprofile == "extended": LOG.info("Selecting extended security profile") serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False) time.sleep(2) serial.send_bytes(stream, "\n", expect_prompt=False, send=False) time.sleep(4)
def run_install_scripts(stream, host_list, aio_type=None, storage=False, release='R6', socks=None, streams=None, labname=None, username='******', password='******', conf_files={}): """ Runs lab install.sh iterations. Currently does not support Simplex systems Args: stream: Stream to controller-0 host_list: list of hosts, used when running aio scripts to install controller-1 at the appropriate time release: Release that is installed. aio_type: Option to run the script for aio setup storage: Option to run the script for storage setup streams: Dictionary of streams to nodes Steps: - Checks for lab_setup files - Runs lab_setup iterations - Unlocks nodes """ LOG.info("Starting to run the second round of lab_setup script. ") if release == 'R6' or release == 'R5' or release == 'R4': ret = serial.send_bytes(stream, '/bin/ls /home/' + username + '/images/ | grep tis-centos-guest.img', prompt="tis-centos-guest.img", fail_ok=True, timeout=10) else: ret = serial.send_bytes(stream, '/bin/ls /home/' + username + '/images/ | grep cgcs-guest.img', prompt="cgcs-guest.img", fail_ok=True, timeout=10) if ret != 0: LOG.info("Guest image not found. Please transfer the file before continuing. " "Press enter once guest image is obtained.") input() time.sleep(5) serial.send_bytes(stream, "chmod +x *.sh", timeout=20) ret = serial.send_bytes(stream, '/bin/ls /home/' + username + '/ | grep lab_setup.sh', prompt="lab_setup.sh", fail_ok=True, timeout=10) if ret != 0: LOG.info("Lab_setup.sh not found. Please transfer the " "required files before continuing. Press enter once files are obtained.") input() conf_str = "" for file in conf_files: conf_str = conf_str + " -f {}".format(file) LOG.info("lab_setup.sh will be run with options: {}".format(conf_str)) start = time.time() if aio_type: ## FOR AIO serial.send_bytes(stream, "source /etc/nova/openrc", prompt='keystone') if release != 'R6': serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False, fail_ok=True) host_helper.check_password(stream, password=password) ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True) if ret != 0: LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. " "Please re-run the iteration before continuing. Press enter to continue.") input() LOG.info("Running system compute-config-complete, " "installation will resume once controller-0 reboots and services are active") serial.send_bytes(stream, "source /etc/nova/openrc", prompt='keystone') serial.send_bytes(stream, "system compute-config-complete", expect_prompt=False) serial.expect_bytes(stream, "login:"******"Installing {} ".format(ctrlr1)) host_helper.install_host(stream, ctrlr1, host_type='controller', host_id=2) # Now wait for controller-1 to come up. Look for "login" on the serial port. # Close the socket if we are done try: serial.expect_bytes(cont1_stream, "ontroller-1 login:"******"{} installation complete".format(ctrlr1)) except Exception as e: LOG.info("Connection failed for {} with {}.".format(ctrlr1, e)) ## Sometimes we get UnicodeDecodeError exception due to the output ## of installation on the serial port, so ignore the exception and try one more time. if HostTimeout.HOST_INSTALL > (time.time()-start_time): LOG.info("Ignore the exception and wait for {} to be installed.".format(ctrlr1)) serial.expect_bytes(cont1_stream, "ontroller-1 login:"******"./lab_setup.sh {}".format(conf_str), expect_prompt=False) host_helper.check_password(stream, password=password) ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True) if ret != 0: LOG.info("Lab_setup.sh failed. Pausing to allow for debugging." " Please re-run the iteration before continuing. Press enter to continue.") input() ## Unlock controller-1 now = time.time() ret = host_helper.unlock_host(stream, ctrlr1) if ret == 1: LOG.info("Cannot unlock controller-1, pausing to allow for debugging. " "Please unlock before continuing. Press enter to continue.") input() retry = 0 while retry < 5: serial.send_bytes(cont1_stream, '\n', expect_prompt=False) try: ret = serial.expect_bytes(cont1_stream, "ontroller-1 login:"******"Unlocking controller-1 timed-out. pausing to allow for debugging. " "Please unlock before continuing. Press enter to continue.") input() break else: LOG.info("Unlocking controller-1 time (mins): {}".format((time.time() - now)/60)) if (time.time() - now)/60 < 15.0: LOG.info("login is found right after host-unlock. Wait and try again.") now = time.time() time.sleep(10) retry += 1 else: break except Exception as e: LOG.info("Unlocking controller-1 failed with {}".format(e)) LOG.info("Pausing to allow for debugging. " "Please unlock before continuing. Press enter to continue.") input() break serial.disconnect(socks[labname + '-' + ctrlr1]) serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False) host_helper.check_password(stream, password=password) ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True) if ret != 0: LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. " "Please re-run the iteration before continuing. Press enter to continue.") input() LOG.info("Completed install successfully.") else: ## FOR NON-AIO cases serial.send_bytes(stream, "source /etc/nova/openrc", prompt='keystone') # TODO (WEI): double check this # Why only if not R5, we run lab_setup.sh here? #if release != 'R5': serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False) host_helper.check_password(stream, password=password) ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True) if ret != 0: LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. " "Please re-run the iteration before continuing. Press enter to continue.") input() ## TODO: ## 1. Unlock sometimes won't bring a node into enabled state. Need to reboot/reset it. ## 2. When unlocking a node, sometimes the node will reboot again after the "login" ## prompt is seen on the serial port, so seeing "login" doesn't mean the node becomes ## "unlocked/enabled". The workaround can be: ## 1) time.sleep(10) and repeat the wait for "login" logic again. ## 2) After seeing "login" prompt, send to controller-0 ## "system host-list | grep node" to look for "enabled". ## Unlock controller-1 now = time.time() ret = host_helper.unlock_host(stream, "controller-1") if ret == 1: LOG.info("Cannot unlock controller-1, pausing to allow for debugging. " "Please unlock before continuing. Press enter to continue.") input() ## Wait for controller-1 to come up retry = 0 ctrlr_1 = labname + "-controller-1" while retry < 5: serial.send_bytes(streams[ctrlr_1], '\n', expect_prompt=False) try: ret = serial.expect_bytes(streams[ctrlr_1], "ontroller-1 login:"******"Unlocking controller-1 timed-out. pausing to allow for debugging. Press enter to continue.") input() break else: LOG.info("Unlock controller-1 time (mins): {}".format((time.time() - now)/60)) ## Sometimes "login" is found right after "host-unlock" is issued, so we need to wait and try again. ## Also sometimes the node will go thru reboot again after "login" is detected on the serial port. ## As the node unlock time is usually longer than 15 mins, as a temperary workaround ## let's wait at least for 15 mins now. if (time.time() - now)/60 < 15.0: LOG.info("Found controller-1 login right away. Need to wait and try again.") now = time.time() time.sleep(10) retry += 1 else: break except Exception as e: LOG.info("Unlock controller-1 failed with {} pausing to allow for debugging. Press enter to continue.".format(e)) input() break serial.disconnect(socks[ctrlr_1]) host_list.remove(ctrlr_1) ## TODO: Well, there is a chance that controller-1 is not "unlocked/enabled" yet. ## See comment #2 above. ## If it is a storage lab if storage: LOG.info("Re-running lab_setup.sh to configure storage nodes.") serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False) host_helper.check_password(stream, password=password) ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True) if ret != 0: LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. " "Please re-run the iteration before continuing. Press enter to continue.") input() now = time.time() for host in host_list: host = host[len(labname)+1:] if host.startswith('storage'): LOG.info("Unlock {}".format(host)) ret = host_helper.unlock_host(stream, host) if ret == 1: LOG.info("Cannot unlock {}, pausing to allow for debugging. " "Please unlock before continuing. Press enter to continue.".format(host)) input() time.sleep(20) ## TODO: Need the retry strategy here too. for host in host_list: if 'storage' in host: serial.send_bytes(streams[host], '\n', expect_prompt=False) try: ret = serial.expect_bytes(streams[host], "{} login:"******"Unlocking {} timed-out. Pause for debugging. Press enter to continue.".format(host)) input() else: LOG.info("Unlocking {} time (mins): {}".format(host, (time.time() - now)/60)) if (time.time() - now)/60 < 15.0: LOG.info("Found {} login too soon. Make sure it is up. Press enter to continue.".format(host)) input() except Exception as e: LOG.info("Unlock {} failed with {}. Pause for debugging. Press enter to continue.".format(host, e)) input() serial.disconnect(socks[host]) host_list.remove(host) LOG.info("Completed storage node unlock") LOG.info("Re-running lab_setup.sh before unlocking compute nodes.") serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False) host_helper.check_password(stream, password=password) ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True) if ret != 0: LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. " "Please re-run the iteration before continuing. Press enter to continue.") input() # unlock compute nodes now = time.time() for host in host_list: host = host[len(labname)+1:] ret = host_helper.unlock_host(stream, host) if ret == 1: LOG.info("Cannot unlock {}, pausing to allow for debugging. " "Please unlock before continuing. Press enter to continue.".format(host)) input() time.sleep(20) LOG.info("Waiting for {} to unlock.".format(host_list)) ## Check unlocking status failed_nodes = [] for host in host_list: # TODO Fix it! 'ogin:' is always found immediately after unlock # WEI: fixed it by retrying retry = 0 while retry < 5: serial.send_bytes(streams[host], '\n', expect_prompt=False) try: ret = serial.expect_bytes(streams[host], "{} login:"******"Unlock {} timed-out.".format(host)) failed_nodes.append(host) break else: LOG.info("Unlock {} time (mins): {}".format(host, (time.time() - now)/60)) if (time.time() - now)/60 < 15.0: LOG.info("login is found right after host-unlock. Wait and try again.") now = time.time() time.sleep(10) retry += 1 else: break except Exception as e: LOG.info("Unlock {} failed with {}".format(host, e)) failed_nodes.append(host) break serial.disconnect(socks[host]) ## Let's reset the VMs that failed to unlock if failed_nodes: vboxmanage.vboxmanage_controlvms(failed_nodes, action="reset") time.sleep(10) tmp_streams = {} tmp_socks = {} LOG.info(failed_nodes) port = 10001 for host in failed_nodes: tmp_sock = serial.connect('{}'.format(host), port) tmp_stream = streamexpect.wrap(tmp_sock, echo=True, close_stream=False) time.sleep(10) tmp_socks[host] = tmp_sock tmp_streams[host] = tmp_stream port += 1 host_failed = False for host in failed_nodes: serial.send_bytes(tmp_streams[host], '\n', expect_prompt=False) try: ret = serial.expect_bytes(tmp_streams[host], "{} login:"******"{} timed-out to become unlocked/available after reset.".format(host)) host_failed = True else: LOG.info("{} became unlocked/available after reset. time (mins): {}".format(host, (time.time() - now)/60)) except Exception as e: LOG.info("{} failed to become unlocked/available after reset with {}".format(host, e)) host_failed = True serial.disconnect(tmp_socks[host]) if host_failed: LOG.info("Not all the nodes are unlocked successfully. Pausing to allow for debugging. " "Once they all become unlocked/enabled/available, press enter to continue.") input() serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False) host_helper.check_password(stream, password=password) ret = serial.expect_bytes(stream, "Done", timeout=HostTimeout.LAB_INSTALL, fail_ok=True) if ret != 0: LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. " "Please re-run the iteration before continuing." " Press enter to continue.") input() LOG.info("Completed lab install.") kpi.LABTIME = time.time()-start LOG.info("Lab install time: {}".format(kpi.LABTIME/60))
def lab_setup_controller_0_locked(stream, username='******', password='******', conf_files={}): """ Runs initial lab_setup when controller-0 is locked. This is for R5 only. Args: stream: Stream to controller-0 Steps: - Checks if controller-0 is locked - Checks for lab_setup files - Runs first lab_setup iteration - Unlocks controller-0 """ time.sleep(10) serial.send_bytes(stream, "source /etc/nova/openrc", prompt='keystone') serial.send_bytes(stream, "system host-list", expect_prompt=False) try: serial.expect_bytes(stream, "locked") except streamexpect.ExpectTimeout: LOG.info("Controller should be locked when configuration is completed.") return 1 ret = serial.send_bytes(stream, '/bin/ls /home/' + username + '/images/ | grep tis-centos-guest.img', prompt="tis-centos-guest.img", fail_ok=True, timeout=10) if ret != 0: LOG.info("Guest image not found. Please transfer the " "required files before continuing. Press enter once files are obtained.") input() time.sleep(5) ret = serial.send_bytes(stream, '/bin/ls /home/' + username + '/ | grep lab_setup.sh', prompt="lab_setup.sh", fail_ok=True, timeout=10) if ret != 0: LOG.info("Lab_setup.sh not found. Please transfer the " "required files before continuing. Press enter once files are obtained.") input() time.sleep(5) conf_str = "" for file in conf_files: conf_str = conf_str + " -f {}".format(file) serial.send_bytes(stream, "sh lab_setup.sh {}".format(conf_str), timeout=HostTimeout.LAB_INSTALL, expect_prompt=False) host_helper.check_password(stream, password=password) ret = serial.expect_bytes(stream, "topping after", timeout=3500, fail_ok=True) if ret != 0: LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. " "Please re-run the iteration before continuing. Press enter to continue.") input() start = time.time() host_helper.unlock_host(stream, 'controller-0') try: ret = serial.expect_bytes(stream, 'login:', timeout=HostTimeout.CONTROLLER_UNLOCK) if ret != 0: LOG.info("Controller-0 not unlocked,Pausing to allow for debugging. " "Please re-run the iteration before continuing. Press enter to continue.") input() except Exception as e: LOG.info("Unlock controller-0 failed with {}. ".format(e)) ## Sometimes we get UnicodeDecodeError exception due to the output ## on the serial port, so ignore that and try one more time. LOG.info("Ignore the exception and wait for controller-0 to be unlocked again.") if HostTimeout.CONTROLLER_UNLOCK > (time.time()-start): serial.expect_bytes(stream, "login:"******"Controller-0 unlock time: {} minutes".format(end)) LOG.info("Waiting for services to activate.") time.sleep(60)