class ControlC(unittest.TestCase): ''' Start a process that does a bunch of console output, and then try and 'control-c' it to stop the process and get a prompt back. ''' CONTROL = 'c' def setUp(self): conf = OpTestConfiguration.conf self.cv_BMC = conf.bmc() self.cv_SYSTEM = conf.system() self.util = OpTestUtil() self.prompt = self.util.build_prompt() def cleanup(self): pass def runTest(self): self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL) console = self.cv_BMC.get_host_console() if (isinstance(self.cv_BMC, OpTestMambo.OpTestMambo)): raise unittest.SkipTest("Mambo so skipping Control-C tests") raw_pty = console.get_console() raw_pty.sendline("find /") time.sleep(2) raw_pty.sendcontrol(self.CONTROL) BMC_DISCONNECT = 'SOL session closed by BMC' timeout = 60 try: rc = raw_pty.expect([BMC_DISCONNECT, self.prompt, pexpect.TIMEOUT, pexpect.EOF], timeout) if rc == 0: raise BMCDisconnected(BMC_DISCONNECT) self.assertEqual(rc, 1, "Failed to find expected prompt") except pexpect.TIMEOUT as e: log.debug(e) log.debug("# TIMEOUT waiting for command to finish with ctrl-c.") log.debug("# Everything is terrible. Fail the world, " "power cycle (if lucky)") self.cv_SYSTEM.set_state(OpSystemState.UNKNOWN_BAD) self.fail("Could not ctrl-c running command in reasonable time") self.cleanup()
class ControlC(unittest.TestCase): CONTROL = 'c' def setUp(self): conf = OpTestConfiguration.conf self.cv_BMC = conf.bmc() self.cv_SYSTEM = conf.system() self.util = OpTestUtil() self.prompt = self.util.build_prompt() def cleanup(self): pass def runTest(self): self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL) console = self.cv_BMC.get_host_console() # I should really make this API less nasty... raw_console = console.get_console() #raw_console.sendline("hexdump -C -v /dev/zero") raw_console.sendline("find /") time.sleep(2) raw_console.sendcontrol(self.CONTROL) BMC_DISCONNECT = 'SOL session closed by BMC' timeout = 60 try: rc = raw_console.expect( [BMC_DISCONNECT, self.prompt, pexpect.TIMEOUT, pexpect.EOF], timeout) if rc == 0: raise BMCDisconnected(BMC_DISCONNECT) self.assertEqual(rc, 1, "Failed to find expected prompt") except pexpect.TIMEOUT as e: log.debug(e) log.debug("# TIMEOUT waiting for command to finish with ctrl-c.") log.debug( "# Everything is terrible. Fail the world, power cycle (if lucky)" ) self.cv_SYSTEM.set_state(OpSystemState.UNKNOWN_BAD) self.fail("Could not ctrl-c running command in reasonable time") self.cleanup()
class HMCConsole(HMCUtil): """ HMCConsole Class Methods to manage the console of LPAR """ def __init__(self, hmc_ip, user_name, password, managed_system, lpar_name, lpar_vios, lpar_prof, lpar_user, lpar_password, block_setup_term=None, delaybeforesend=None, timeout_factor=1, logfile=sys.stdout, prompt=None, scratch_disk="", check_ssh_keys=False, known_hosts_file=None, proxy=""): self.logfile = logfile self.hmc_ip = hmc_ip self.user = user_name self.passwd = password self.mg_system = managed_system self.util = OpTestUtil() self.expect_prompt = self.util.build_prompt(prompt) + "$" self.lpar_name = lpar_name self.lpar_vios = lpar_vios self.lpar_prof = lpar_prof self.lpar_user = lpar_user self.lpar_password = lpar_password self.scratch_disk = scratch_disk self.proxy = proxy self.state = ConsoleState.DISCONNECTED self.delaybeforesend = delaybeforesend self.system = None # OpTestUtil instance is NOT conf's self.prompt = prompt self.pty = None self.delaybeforesend = delaybeforesend # allows caller specific control of when to block setup_term self.block_setup_term = block_setup_term # tells setup_term to not throw exceptions, like when system off self.setup_term_quiet = 0 # flags the object to abandon setup_term operations, like when system off self.setup_term_disable = 0 # FUTURE - System Console currently tracked in System Object # state tracking, reset on boot and state changes self.PS1_set = -1 self.LOGIN_set = -1 self.SUDO_set = -1 super(HMCConsole, self).__init__(hmc_ip, user_name, password, scratch_disk, proxy, logfile, managed_system, lpar_name, prompt, block_setup_term, delaybeforesend, timeout_factor, lpar_prof, lpar_vios, lpar_user, lpar_password, check_ssh_keys, known_hosts_file) def set_system(self, system): self.ssh.set_system(system) self.system = system self.pty = self.get_console() self.pty.set_system(system) def get_host_console(self): return self.pty def set_system_setup_term(self, flag): self.system.block_setup_term = flag def get_system_setup_term(self): return self.system.block_setup_term def get_scratch_disk(self): return self.scratch_disk def get_proxy(self): return self.proxy def hostname(self): return self.hmc_ip def username(self): return self.user def password(self): return self.passwd def set_block_setup_term(self, flag): self.block_setup_term = flag def get_block_setup_term(self): return self.block_setup_term def enable_setup_term_quiet(self): self.setup_term_quiet = 1 self.setup_term_disable = 0 def disable_setup_term_quiet(self): self.setup_term_quiet = 0 self.setup_term_disable = 0 def close(self): self.util.clear_state(self) try: self.pty.close() if self.pty.status != -1: # leaving for debug if os.WIFEXITED(self.pty.status): os.WEXITSTATUS(self.pty.status) else: os.WTERMSIG(self.pty.status) self.state = ConsoleState.DISCONNECTED except pexpect.ExceptionPexpect: self.state = ConsoleState.DISCONNECTED raise "HMC Console: failed to close console" except Exception: self.state = ConsoleState.DISCONNECTED log.debug("HMC close -> TERMINATE") def connect(self, logger=None): if self.state == ConsoleState.CONNECTED: return self.pty self.util.clear_state(self) # clear when coming in DISCONNECTED log.info("De-activating the console") self.deactivate_lpar_console() log.debug("#HMC Console CONNECT") command = "sshpass -p %s ssh -p 22 -l %s %s -o PubkeyAuthentication=no"\ " -o afstokenpassing=no -q -o 'UserKnownHostsFile=/dev/null'"\ " -o 'StrictHostKeyChecking=no'" try: self.pty = Spawn(command % (self.passwd, self.user, self.hmc_ip)) log.info("Opening the LPAR console") time.sleep(STALLTIME) self.pty.send('\r') self.pty.sendline("mkvterm -m %s -p %s" % (self.mg_system, self.lpar_name)) self.pty.send('\r') time.sleep(STALLTIME) i = self.pty.expect(["Open Completed.", pexpect.TIMEOUT], timeout=60) self.pty.logfile = sys.stdout if logger: self.pty.logfile_read = OpTestLogger.FileLikeLogger(logger) else: self.pty.logfile_read = OpTestLogger.FileLikeLogger(log) if i == 0: time.sleep(STALLTIME) self.state = ConsoleState.CONNECTED self.pty.setwinsize(1000, 1000) else: raise OpTestError("Check the lpar activate command") except Exception as exp: self.state = ConsoleState.DISCONNECTED raise CommandFailed( 'OPexpect.spawn', 'OPexpect.spawn encountered a problem: ' + str(exp), -1) if self.delaybeforesend: self.pty.delaybeforesend = self.delaybeforesend if not self.pty.isalive(): raise CommandFailed("mkvterm", self.pty.read(), self.pty.status) return self.pty def check_state(self): return self.state def get_console(self, logger=None): if self.state == ConsoleState.DISCONNECTED: self.util.clear_state(self) self.connect(logger=logger) time.sleep(STALLTIME) l_rc = self.pty.expect(["login:"******"Shell prompt changed") else: self.pty.send('\r') log.debug("Waiting till booting!") self.pty = self.get_login_prompt() if self.system.SUDO_set != 1 or self.system.LOGIN_set != 1 or self.system.PS1_set != 1: self.util.setup_term(self.system, self.pty, None, self.system.block_setup_term) # Clear buffer before usage self.pty.buffer = "" return self.pty def get_login_prompt(self): # Assuming 'Normal' boot set in LPAR profile # We wait for upto 500 seconds for LPAR to boot to OS self.pty.send('\r') time.sleep(STALLTIME) log.debug("Waiting for login screen") i = self.pty.expect(["login:"******"System has booted") time.sleep(STALLTIME) elif i == 1: log.debug("Console already logged in") else: log.error("Failed to get login prompt %s", self.pty.before) # To cheat system for making using of HMC SSH self.system.PS1_set = 1 self.system.LOGIN_set = 1 self.system.SUDO_set = 1 return self.pty def run_command(self, i_cmd, timeout=15): return self.util.run_command(self, i_cmd, timeout)
class HMCUtil(): ''' Utility and functions of HMC object ''' def __init__(self, hmc_ip, user_name, password, scratch_disk="", proxy="", logfile=sys.stdout, managed_system=None, lpar_name=None, prompt=None, block_setup_term=None, delaybeforesend=None, timeout_factor=None, lpar_prof=None, lpar_vios=None, lpar_user=None, lpar_password=None, check_ssh_keys=False, known_hosts_file=None, tgt_managed_system=None): self.hmc_ip = hmc_ip self.user = user_name self.passwd = password self.logfile = logfile self.mg_system = managed_system self.tgt_mg_system = tgt_managed_system self.check_ssh_keys = check_ssh_keys self.known_hosts_file = known_hosts_file self.lpar_name = lpar_name self.lpar_prof = lpar_prof self.lpar_user = lpar_user self.lpar_password = lpar_password self.lpar_vios = lpar_vios self.util = OpTestUtil() self.prompt = prompt self.expect_prompt = self.util.build_prompt(prompt) + "$" self.ssh = OpTestSSH(hmc_ip, user_name, password, logfile=self.logfile, check_ssh_keys=check_ssh_keys, known_hosts_file=known_hosts_file, block_setup_term=block_setup_term) self.scratch_disk = scratch_disk self.proxy = proxy self.scratch_disk_size = None self.delaybeforesend = delaybeforesend self.system = None # OpTestUtil instance is NOT conf's self.pty = None # allows caller specific control of when to block setup_term self.block_setup_term = block_setup_term # tells setup_term to not throw exceptions, like when system off self.setup_term_quiet = 0 # flags the object to abandon setup_term operations, like when system off self.setup_term_disable = 0 # functional simulators are very slow, so multiply all default timeouts by this factor self.timeout_factor = timeout_factor # state tracking, reset on boot and state changes # console tracking done on System object for the system console self.PS1_set = -1 self.LOGIN_set = -1 self.SUDO_set = -1 def deactivate_lpar_console(self): self.ssh.run_command("rmvterm -m %s -p %s" % (self.mg_system, self.lpar_name), timeout=10) def poweroff_system(self): if self.get_system_state() != OpManagedState.OPERATING: raise OpTestError('Managed Systen not in Operating state') self.ssh.run_command("chsysstate -m %s -r sys -o off" % self.mg_system) self.wait_system_state(OpManagedState.OFF) def poweron_system(self): if self.get_system_state() != OpManagedState.OFF: raise OpTestError('Managed Systen not is Power off state!') self.ssh.run_command("chsysstate -m %s -r sys -o on" % self.mg_system) self.wait_system_state() if self.lpar_vios: log.debug("Starting VIOS %s", self.lpar_vios) self.poweron_lpar(vios=True) def poweroff_lpar(self): if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]: log.info('LPAR Already powered-off!') return self.ssh.run_command( "chsysstate -m %s -r lpar -n %s -o shutdown --immed" % (self.mg_system, self.lpar_name)) self.wait_lpar_state(OpHmcState.NOT_ACTIVE) def poweron_lpar(self, vios=False): if self.get_lpar_state(vios) == OpHmcState.RUNNING: log.info('LPAR Already powered on!') return BMC_CONST.FW_SUCCESS lpar_name = self.lpar_name if vios: lpar_name = self.lpar_vios cmd = "chsysstate -m %s -r lpar -n %s -o on" % (self.mg_system, lpar_name) if self.lpar_prof: cmd = "%s -f %s" % (cmd, self.lpar_prof) self.wait_lpar_state(OpHmcState.NOT_ACTIVE, vios=vios) self.ssh.run_command(cmd) self.wait_lpar_state(vios=vios) time.sleep(STALLTIME) return BMC_CONST.FW_SUCCESS def dumprestart_lpar(self): if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]: log.info('LPAR Already powered-off!') return self.ssh.run_command("chsysstate -m %s -r lpar -n %s -o dumprestart" % (self.mg_system, self.lpar_name)) self.wait_lpar_state() def restart_lpar(self): if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]: log.info('LPAR Already powered-off!') return self.ssh.run_command( "chsysstate -m %s -r lpar -n %s -o shutdown --immed --restart" % (self.mg_system, self.lpar_name)) self.wait_lpar_state() def get_lpar_state(self, vios=False): lpar_name = self.lpar_name if vios: lpar_name = self.lpar_vios state = self.ssh.run_command( 'lssyscfg -m %s -r lpar --filter lpar_names=%s -F state' % (self.mg_system, lpar_name))[-1] ref_code = self.ssh.run_command( 'lsrefcode -m %s -r lpar --filter lpar_names=%s -F refcode' % (self.mg_system, lpar_name))[-1] if state == 'Running': if 'Linux' in ref_code or not ref_code: return 'Running' else: return 'Booting' return state def get_system_state(self): state = self.ssh.run_command('lssyscfg -m %s -r sys -F state' % self.mg_system) return state[-1] def wait_lpar_state(self, exp_state=OpHmcState.RUNNING, vios=False, timeout=WAITTIME): state = self.get_lpar_state(vios) count = 0 while state != exp_state: state = self.get_lpar_state(vios) log.info("Current state: %s", state) time.sleep(timeout) count = 1 if count > 120: raise OpTestError("Time exceeded for reaching %s" % exp_state) def wait_system_state(self, exp_state=OpManagedState.OPERATING, timeout=WAITTIME): state = self.get_system_state() count = 0 while state != exp_state: state = self.get_system_state() log.info("Current state: %s", state) time.sleep(timeout) count = 1 if count > 60: raise OpTestError("Time exceeded for reaching %s" % exp_state) def is_lpar_in_managed_system(self, mg_system=None): lpar_list = self.ssh.run_command('lssyscfg -r lpar -m %s -F name' % mg_system) if self.lpar_name in lpar_list: log.info("%s lpar found in managed system %s" % (mg_system, self.lpar_name)) return True return False def migrate_lpar(self, src_mg_system=None, dest_mg_system=None): if src_mg_system == None or dest_mg_system == None: raise OpTestError( "Source and Destination Managed System required for LPM") if not self.is_lpar_in_managed_system(src_mg_system): raise OpTestError("Lpar %s not found in managed system %s" % (self.lpar_name, src_mg_system)) self.ssh.run_command('migrlpar -o v -m %s -t %s -p %s' % (src_mg_system, dest_mg_system, self.lpar_name)) self.ssh.run_command('migrlpar -o m -m %s -t %s -p %s' % (src_mg_system, dest_mg_system, self.lpar_name)) if self.is_lpar_in_managed_system(dest_mg_system): log.info("Migration of lpar %s from %s to %s is successfull" % (self.lpar_name, src_mg_system, dest_mg_system)) self.mg_system = dest_mg_system return True log.info("Migration of lpar %s from %s to %s failed" % (self.lpar_name, src_mg_system, dest_mg_system)) return False def run_command_ignore_fail(self, command, timeout=60, retry=0): return self.ssh.run_command_ignore_fail(command, timeout * self.timeout_factor, retry) def run_command(self, i_cmd, timeout=15): return self.ssh.run_command(i_cmd, timeout)
class OpTestHMC(): ''' This class contains the modules to perform various HMC operations on an LPAR. The Host IP, username and password of HMC have to be passed to the class intially while creating the object for the class. ''' def __init__(self, hmc_ip, user_name, password, scratch_disk="", proxy="", logfile=sys.stdout, managed_system=None, lpar_name=None, prompt=None, lpar_prof=None, lpar_vios=None, lpar_user=None, lpar_password=None, check_ssh_keys=False, known_hosts_file=None): self.hmc_ip = hmc_ip self.user = user_name self.passwd = password self.logfile = logfile self.system = managed_system self.system = managed_system self.check_ssh_keys = check_ssh_keys self.known_hosts_file = known_hosts_file self.lpar_name = lpar_name self.lpar_prof = lpar_prof self.lpar_user = lpar_user self.lpar_password = lpar_password self.lpar_vios = lpar_vios self.lpar_con = None self.vterm = False self.util = OpTestUtil() self.prompt = prompt self.expect_prompt = self.util.build_prompt(prompt) + "$" self.conf = OpTestConfiguration.conf self.ssh = OpTestSSH(hmc_ip, user_name, password, logfile=self.logfile, check_ssh_keys=check_ssh_keys, known_hosts_file=known_hosts_file) self.scratch_disk = scratch_disk self.proxy = proxy self.scratch_disk_size = None def hostname(self): return self.hmc_ip def username(self): return self.user def password(self): return self.passwd def set_system(self, system): self.ssh.set_system(system) def get_scratch_disk(self): return self.scratch_disk def run_command(self, i_cmd, timeout=15): return self.ssh.run_command(i_cmd, timeout) def deactivate_lpar_console(self): self.run_command("rmvterm -m %s -p %s" % (self.system, self.lpar_name), timeout=10) def poweroff_system(self): if self.get_system_state() != OpManagedState.OPERATING: raise OpTestError('Managed Systen not in Operating state') self.run_command("chsysstate -m %s -r sys -o off" % self.system) self.wait_system_state(OpManagedState.OFF) def poweron_system(self): if self.get_system_state() != OpManagedState.OFF: raise OpTestError('Managed Systen not is Power off state!') self.run_command("chsysstate -m %s -r sys -o on" % self.system) self.wait_system_state() if self.lpar_vios: log.debug("Starting VIOS %s" % self.lpar_vios) self.poweron_lpar(vios=True) def poweroff_lpar(self): if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]: log.info('LPAR Already powered-off!') return self.run_command("chsysstate -m %s -r lpar -n %s -o shutdown --immed" % (self.system, self.lpar_name)) self.wait_lpar_state(OpHmcState.NOT_ACTIVE) def poweron_lpar(self, runtime=False, vios=False): if self.get_lpar_state(vios) == OpHmcState.RUNNING: log.info('LPAR Already powered on!') return lpar_name = self.lpar_name if vios: lpar_name = self.lpar_vios cmd = "chsysstate -m %s -r lpar -n %s -o on" % (self.system, lpar_name) if not vios: if self.lpar_prof: cmd = "%s -f %s" % (cmd, self.lpar_prof) self.wait_lpar_state(OpHmcState.NOT_ACTIVE, vios=vios) self.run_command(cmd) self.wait_lpar_state(vios=vios) if runtime: self.wait_login_prompt(self.get_console_prompt()) self.close_console(self.lpar_con) def get_lpar_state(self, vios=False): lpar_name = self.lpar_name if vios: lpar_name = self.lpar_vios state = self.run_command( 'lssyscfg -m %s -r lpar --filter lpar_names=%s -F state' % (self.system, lpar_name)) return state[-1] def get_system_state(self): state = self.run_command('lssyscfg -m %s -r sys -F state' % self.system) return state[-1] def wait_lpar_state(self, exp_state=OpHmcState.RUNNING, vios=False, timeout=WAITTIME): state = self.get_lpar_state(vios) count = 0 while state != exp_state: state = self.get_lpar_state(vios) log.info("Current state: %s" % state) time.sleep(timeout) count = 1 if count > 120: raise OpTestError("Time exceeded for reaching %s" % exp_state) def wait_system_state(self, exp_state=OpManagedState.OPERATING, timeout=WAITTIME): state = self.get_system_state() count = 0 while state != exp_state: state = self.get_system_state() log.info("Current state: %s" % state) time.sleep(timeout) count = 1 if count > 60: raise OpTestError("Time exceeded for reaching %s" % exp_state) def vterm_run_command(self, console, cmd, timeout=60): if console is None: console = self.get_console() try: console.send(cmd) console.send('\r') output = console.before console.sendline("echo $?") console.send('\r') # Time to accumulate both outputs time.sleep(STALLTIME) rc = console.expect([self.expect_prompt], timeout=timeout) if rc != 0: exitcode = int(console.before) log.debug("# LAST COMMAND EXIT CODE %d (%s)" % (exitcode, repr(console.before))) except pexpect.TIMEOUT as e: log.debug(e) log.debug("# TIMEOUT waiting for command to finish.") log.debug("# Attempting to control-c") try: console.sendcontrol('c') rc = console.expect([self.expect_prompt], 10) if rc == 0: raise CommandFailed(cmd, "TIMEOUT", -1) except pexpect.TIMEOUT: log.error("# Timeout trying to kill timed-out command.") log.error( "# Failing current command and attempting to continue") self.deactivate_lpar_console() raise CommandFailed("console", "timeout", -1) raise e if rc == 0: res = console.before res = res.split(cmd) return res[-1].splitlines()[1:-2] else: res = output res = res.splitlines() if exitcode != 0: raise CommandFailed(cmd, res, exitcode) return res def get_console(self): console = self.get_console_prompt() time.sleep(STALLTIME) l_rc = console.expect(["login:"******"Shell prompt changed") else: console.send('\r') log.debug("Waiting till booting!") console = self.wait_login_prompt(console, self.lpar_user, self.lpar_password) return console def check_vterm(self): return self.vterm def wait_login_prompt(self, console, username=None, password=None): if not console: raise OpTestError("Console is not provided") # Assuming 'Normal' boot set in LPAR profile # We wait for upto 500 seconds for LPAR to boot to OS console.buffer = "" console.send('\r') time.sleep(STALLTIME) log.debug("Waiting for login screen") i = console.expect(["login:"******"System has booted") if username and password: self.lpar_con.sendline(username) self.lpar_con.send('\r') time.sleep(STALLTIME) i = self.lpar_con.expect(["Password:"******"Last login", "incorrect", pexpect.TIMEOUT], timeout=60) if i == 0: log.info('Logged in to console') time.sleep(STALLTIME) return self.lpar_con elif i == 1: raise OpTestError("Wrong Credentials for host") else: raise OpTestError("Upexpected return") time.sleep(STALLTIME) return self.lpar_con elif i == 1: # Assuming console already logged in !" return self.lpar_con else: log.error("%s %s" % (i, self.lpar_con.before)) raise OpTestError("Console in different state") def get_console_prompt(self): if self.get_lpar_state() != OpHmcState.RUNNING: raise OpTestError('LPAR is not in Running State. Please check!') log.info("De-activating the console") self.deactivate_lpar_console() command = "sshpass -p %s ssh -p 22 -l %s %s -o PubkeyAuthentication=no"\ " -o afstokenpassing=no -q -o 'UserKnownHostsFile=/dev/null'"\ " -o 'StrictHostKeyChecking=no'" self.lpar_con = OPexpect.spawn(command % (self.passwd, self.user, self.hmc_ip)) self.wait_lpar_state() log.info("Opening the LPAR console") self.vterm = True self.lpar_con.sendline("mkvterm -m %s -p %s" % (self.system, self.lpar_name)) self.lpar_con.send('\r') time.sleep(STALLTIME) i = self.lpar_con.expect(["Open Completed.", pexpect.TIMEOUT], timeout=60) self.lpar_con.logfile = sys.stdout self.lpar_con.logfile_read = OpTestLogger.FileLikeLogger(log) if i == 0: time.sleep(STALLTIME) return self.lpar_con else: raise OpTestError("Check the lpar activate command") def close_console(self, console): console.send('~.') self.vterm = False time.sleep(STALLTIME) console.close()
class HMCUtil(): ''' Utility and functions of HMC object ''' def __init__(self, hmc_ip, user_name, password, scratch_disk="", proxy="", logfile=sys.stdout, managed_system=None, lpar_name=None, prompt=None, block_setup_term=None, delaybeforesend=None, timeout_factor=None, lpar_prof=None, lpar_vios=None, lpar_user=None, lpar_password=None, check_ssh_keys=False, known_hosts_file=None, tgt_managed_system=None, tgt_lpar=None): self.hmc_ip = hmc_ip self.user = user_name self.passwd = password self.logfile = logfile self.mg_system = managed_system self.tgt_mg_system = tgt_managed_system self.tgt_lpar = tgt_lpar self.check_ssh_keys = check_ssh_keys self.known_hosts_file = known_hosts_file self.lpar_name = lpar_name self.lpar_prof = lpar_prof self.lpar_user = lpar_user self.lpar_password = lpar_password self.lpar_vios = lpar_vios self.util = OpTestUtil() self.prompt = prompt self.expect_prompt = self.util.build_prompt(prompt) + "$" self.ssh = OpTestSSH(hmc_ip, user_name, password, logfile=self.logfile, check_ssh_keys=check_ssh_keys, known_hosts_file=known_hosts_file, block_setup_term=block_setup_term) self.scratch_disk = scratch_disk self.proxy = proxy self.scratch_disk_size = None self.delaybeforesend = delaybeforesend self.system = None # OpTestUtil instance is NOT conf's self.pty = None # allows caller specific control of when to block setup_term self.block_setup_term = block_setup_term # tells setup_term to not throw exceptions, like when system off self.setup_term_quiet = 0 # flags the object to abandon setup_term operations, like when system off self.setup_term_disable = 0 # functional simulators are very slow, so multiply all default timeouts by this factor self.timeout_factor = timeout_factor # state tracking, reset on boot and state changes # console tracking done on System object for the system console self.PS1_set = -1 self.LOGIN_set = -1 self.SUDO_set = -1 def deactivate_lpar_console(self): ''' Deactivate/disconnect the LPAR Console ''' self.ssh.run_command("rmvterm -m %s -p %s" % (self.mg_system, self.lpar_name), timeout=10) def poweroff_system(self): ''' PowerOFF the managed system ''' if self.get_system_state() != OpManagedState.OPERATING: raise OpTestError('Managed Systen not in Operating state') self.ssh.run_command("chsysstate -m %s -r sys -o off" % self.mg_system) self.wait_system_state(OpManagedState.OFF) def poweron_system(self): ''' PowerON the managed system ''' if self.get_system_state() != OpManagedState.OFF: raise OpTestError('Managed Systen not is Power off state!') self.ssh.run_command("chsysstate -m %s -r sys -o on" % self.mg_system) self.wait_system_state() if self.lpar_vios: log.debug("Starting VIOS %s", self.lpar_vios) self.poweron_lpar(vios=True) def poweroff_lpar(self, remote_hmc=None): ''' PowerOFF the LPAR :param remote_hmc: object, remote HMC instance ''' hmc = remote_hmc if remote_hmc else self if self.get_lpar_state(remote_hmc=remote_hmc) in [ OpHmcState.NOT_ACTIVE, OpHmcState.NA ]: log.info('LPAR Already powered-off!') return hmc.ssh.run_command( "chsysstate -m %s -r lpar -n %s -o shutdown --immed" % (hmc.mg_system, self.lpar_name)) self.wait_lpar_state(OpHmcState.NOT_ACTIVE, remote_hmc=remote_hmc) def poweron_lpar(self, vios=False, remote_hmc=None): ''' PowerON the LPAR :param vios: Boolean, to identify VIOS partition :param remote_hmc: object, remote HMC instance :returns: BMC_CONST.FW_SUCCESS up on success ''' hmc = remote_hmc if remote_hmc else self if self.get_lpar_state(vios, remote_hmc=remote_hmc) == OpHmcState.RUNNING: log.info('LPAR Already powered on!') return BMC_CONST.FW_SUCCESS lpar_name = self.lpar_name if vios: lpar_name = hmc.lpar_vios cmd = "chsysstate -m %s -r lpar -n %s -o on" % (hmc.mg_system, lpar_name) if self.lpar_prof: cmd = "%s -f %s" % (cmd, self.lpar_prof) self.wait_lpar_state(OpHmcState.NOT_ACTIVE, vios=vios, remote_hmc=remote_hmc) hmc.ssh.run_command(cmd) self.wait_lpar_state(vios=vios, remote_hmc=remote_hmc) time.sleep(STALLTIME) return BMC_CONST.FW_SUCCESS def dumprestart_lpar(self): ''' Capture dump and restart the LPAR ''' if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]: log.info('LPAR Already powered-off!') return self.ssh.run_command("chsysstate -m %s -r lpar -n %s -o dumprestart" % (self.mg_system, self.lpar_name)) self.wait_lpar_state() def restart_lpar(self): ''' Restart LPAR immediately ''' if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]: log.info('LPAR Already powered-off!') return self.ssh.run_command( "chsysstate -m %s -r lpar -n %s -o shutdown --immed --restart" % (self.mg_system, self.lpar_name)) self.wait_lpar_state() def get_lpar_cfg(self): ''' Get LPAR configuration parameters :returns: LPAR configuration parameters in key, value pair ''' out = self.ssh.run_command( "lssyscfg -r prof -m %s --filter 'lpar_names=%s'" % (self.mg_system, self.lpar_name))[-1] cfg_dict = {} splitter = shlex.shlex(out) splitter.whitespace += ',' splitter.whitespace_split = True for values in list(splitter): data = values.split("=") key = data[0] value = data[1] cfg_dict[key] = value return cfg_dict def set_lpar_cfg(self, arg_str): ''' Set LPAR configuration parameter values :param arg_str: configuration values in key, value pair seperated by comma ''' if not self.lpar_prof: raise OpTestError("Profile needs to be defined to use this method") self.ssh.run_command( "chsyscfg -r prof -m %s -p %s -i 'lpar_name=%s,name=%s,%s' --force" % (self.mg_system, self.lpar_name, self.lpar_name, self.lpar_prof, arg_str)) def get_lpar_state(self, vios=False, remote_hmc=None): ''' Get current state of LPAR :param vios: Boolean, to identify VIOS partition :param remote_hmc: object, remote HMC instance :returns: the current status of the LPAR e.g 'Running' or 'Booting' ''' hmc = remote_hmc if remote_hmc else self lpar_name = self.lpar_name if vios: lpar_name = hmc.lpar_vios state = hmc.ssh.run_command( 'lssyscfg -m %s -r lpar --filter lpar_names=%s -F state' % (hmc.mg_system, lpar_name))[-1] ref_code = hmc.ssh.run_command( 'lsrefcode -m %s -r lpar --filter lpar_names=%s -F refcode' % (hmc.mg_system, lpar_name))[-1] if state == 'Running': if 'Linux' in ref_code or not ref_code: return 'Running' else: return 'Booting' return state def get_system_state(self): ''' Get current state of managed system :returns: the current status of the managed system ''' state = self.ssh.run_command('lssyscfg -m %s -r sys -F state' % self.mg_system) return state[-1] def wait_lpar_state(self, exp_state=OpHmcState.RUNNING, vios=False, timeout=WAITTIME, remote_hmc=None): ''' Wait for a particular state of LPAR :param exp_state: constant, expected HMC STATE, default is OpHmcState.RUNNING :param vios: Boolean, to identify VIOS partition :param timeout: number, Wait time in seconds :param remote_hmc: object, remote HMC instance :raises: :class:`common.OpTestError` when the timeout happens ''' state = self.get_lpar_state(vios, remote_hmc=remote_hmc) count = 0 while state != exp_state: state = self.get_lpar_state(vios, remote_hmc=remote_hmc) log.info("Current state: %s", state) time.sleep(timeout) count += 1 if count > 120: raise OpTestError("Time exceeded for reaching %s" % exp_state) def wait_system_state(self, exp_state=OpManagedState.OPERATING, timeout=SYS_WAITTIME): ''' Wait for a particular state of managed system :param exp_state: constatn, expected STATE, default is OpManagedState.OPERATING :param timeout: number, Wait time in seconds :raises: :class:`common.OpTestError` when the timeout happens ''' state = self.get_system_state() count = 0 while state != exp_state: state = self.get_system_state() log.info("Current state: %s", state) time.sleep(timeout) count += 1 if count > 60: raise OpTestError("Time exceeded for reaching %s" % exp_state) def is_lpar_in_managed_system(self, mg_system=None, lpar_name=None, remote_hmc=None): ''' Check if LPAR is available on a managed system :param mg_system: string, managed system name :param lpar_name: string, LPAR name :param remote_hmc: object, remote HMC instance :returns: True - when the LPAR is in managed system False - when the LPAR is not in managed system ''' hmc = remote_hmc if remote_hmc else self lpar_list = hmc.ssh.run_command('lssyscfg -r lpar -m %s -F name' % mg_system) if lpar_name in lpar_list: log.info("%s lpar found in managed system %s" % (lpar_name, mg_system)) return True return False def migrate_lpar(self, src_mg_system=None, dest_mg_system=None, options=None, param="", timeout=300): ''' Migrate the LPAR from source managed system to the destination managed system :param src_mg_system: string, source managed system name :param dest_mg_system: string, destination manages system name :param options: string, options for migration :param param: string, required parameters for migration :param timeout: number, time out value in seconds, default 300 :returns: True - when LPAR migration success False - when LPAR migration unsuccess ''' if src_mg_system == None or dest_mg_system == None: raise OpTestError( "Source and Destination Managed System required for LPM") if not self.is_lpar_in_managed_system(src_mg_system, self.lpar_name): raise OpTestError("Lpar %s not found in managed system %s" % (self.lpar_name, src_mg_system)) for mode in ['v', 'm']: cmd = 'migrlpar -o %s -m %s -t %s -p %s %s' % ( mode, src_mg_system, dest_mg_system, self.lpar_name, param) if options: cmd = "%s %s" % (cmd, options) self.ssh.run_command(cmd, timeout=timeout) log.debug("Waiting for %.2f minutes." % (timeout / 60)) time.sleep(timeout) if self.is_lpar_in_managed_system(dest_mg_system, self.lpar_name): cmd = "lssyscfg -m %s -r lpar --filter lpar_names=%s -F state" % ( dest_mg_system, self.lpar_name) lpar_state = self.ssh.run_command(cmd)[0] if lpar_state not in [ 'Migrating - Running', 'Migrating - Not Activated' ]: log.info("Migration of lpar %s from %s to %s is successfull" % (self.lpar_name, src_mg_system, dest_mg_system)) self.mg_system = dest_mg_system return True self.recover_lpar(src_mg_system, dest_mg_system, stop_lpm=True, timeout=timeout) log.info("Migration of lpar %s from %s to %s failed" % (self.lpar_name, src_mg_system, dest_mg_system)) return False def set_ssh_key_auth(self, hmc_ip, hmc_user, hmc_passwd, remote_hmc=None): ''' Set SSH authentication keys :param hmc_ip: number, in ip address format :param hmc_user: string, HMC user name :param hmc_passwd: string, HMC user password :param remote_hmc: object, remote HMC instance :raises: `CommandFailed` ''' hmc = remote_hmc if remote_hmc else self try: cmd = "mkauthkeys -u %s --ip %s --test" % (hmc_user, hmc_ip) hmc.run_command(cmd, timeout=120) except CommandFailed: try: cmd = "mkauthkeys -u %s --ip %s --passwd %s" % ( hmc_user, hmc_ip, hmc_passwd) hmc.run_command(cmd, timeout=120) except CommandFailed as cf: raise cf def cross_hmc_migration(self, src_mg_system=None, dest_mg_system=None, target_hmc_ip=None, target_hmc_user=None, target_hmc_passwd=None, remote_hmc=None, options=None, param="", timeout=300): ''' migration of LPAR across HMC :param src_mg_system: string, Source managed system :param dest_mg_system: string, Destination managed system :param target_hmc_ip: number, in IP address format :param target_hmc_user: string, target HMC user name :param target_hmc_passwd: string, target HMC user password :param remote_hmc: object, remote HMC instance :param options: string, migration options :param param: string, migration parameters :param timeout: number, time in seconds ''' hmc = remote_hmc if remote_hmc else self if src_mg_system == None or dest_mg_system == None: raise OpTestError("Source and Destination Managed System "\ "required for Cross HMC LPM") if target_hmc_ip == None or target_hmc_user == None or target_hmc_passwd == None: raise OpTestError("Destination HMC IP, Username, and Password "\ "required for Cross HMC LPM") if not self.is_lpar_in_managed_system(src_mg_system, self.lpar_name, remote_hmc): raise OpTestError("Lpar %s not found in managed system %s" % (self.lpar_name, src_mg_system)) self.set_ssh_key_auth(target_hmc_ip, target_hmc_user, target_hmc_passwd, remote_hmc) for mode in ['v', 'm']: cmd = "migrlpar -o %s -m %s -t %s -p %s -u %s --ip %s %s" % ( mode, src_mg_system, dest_mg_system, self.lpar_name, target_hmc_user, target_hmc_ip, param) if options: cmd = "%s %s" % (cmd, options) hmc.ssh.run_command(cmd, timeout=timeout) def recover_lpar(self, src_mg_system, dest_mg_system, stop_lpm=False, timeout=300): ''' Recover LPAR :param src_mg_system: string, Source managed system :param dest_mg_system: string, Destination managed system :param stop_lpm: Boolean, to stop migration, default False :param timeout: number, time in seconds :returns: True - when LPAR recovery success False - when LPAR recovery unsuccess ''' if stop_lpm: self.ssh.run_command("migrlpar -o s -m %s -p %s" % (src_mg_system, self.lpar_name), timeout=timeout) self.ssh.run_command("migrlpar -o r -m %s -p %s" % (src_mg_system, self.lpar_name), timeout=timeout) if not self.is_lpar_in_managed_system(dest_mg_system, self.lpar_name): log.info("LPAR recovered at managed system %s" % src_mg_system) return True log.info("LPAR failed to recover at managed system %s" % src_mg_system) return False def get_adapter_id(self, mg_system, loc_code, remote_hmc=None): ''' Get SRIOV adapter id :param mg_system: string, Source managed system :param loc_code: string, adapater :param remote_hmc: object, remote HMC instance :returns: string, adapter id ''' hmc = remote_hmc if remote_hmc else self cmd = 'lshwres -m {} -r sriov --rsubtype adapter -F phys_loc:adapter_id'.format( mg_system) adapter_id_output = hmc.ssh.run_command(cmd) for line in adapter_id_output: if str(loc_code) in line: return line.split(':')[1] return '' def get_lpar_id(self, mg_system, l_lpar_name, remote_hmc=None): ''' Get LPAR id :param mg_system: string, Source managed system :param l_lpar_name: string, LPAR name :param remote_hmc: object, remote HMC instance :returns: number, LPAR id on success, 0 on failure ''' hmc = remote_hmc if remote_hmc else self cmd = 'lssyscfg -m %s -r lpar --filter lpar_names=%s -F lpar_id' % ( mg_system, l_lpar_name) lpar_id_output = hmc.ssh.run_command(cmd) for line in lpar_id_output: if l_lpar_name in line: return 0 return line return 0 def is_msp_enabled(self, mg_system, vios_name, remote_hmc=None): ''' The function checks if the moving service option is enabled on the given lpar partition. :param mg_system: string, Source managed system :param vios_name: string, VIOS name :param remote_hmc: object, remote HMC instance :returns: Boolean, True on success, False on failure ''' hmc = remote_hmc if remote_hmc else self cmd = "lssyscfg -m %s -r lpar --filter lpar_names=%s -F msp" % ( mg_system, vios_name) msp_output = hmc.ssh.run_command(cmd) if int(msp_output[0]) != 1: return False return True def gather_logs(self, list_of_commands=[], remote_hmc=None, output_dir=None): ''' Gather the logs for the commands at the given directory :param list_of_commands: list, list of commands :param remote_hmc: object, remote HMC instance :param output_dir: string, dir to store logs :returns: Boolean, True on success, False on failure :raises: `CommandFailed` ''' hmc = remote_hmc if remote_hmc else self if not output_dir: output_dir = "HMC_Logs_%s" % (time.asctime( time.localtime())).replace(" ", "_") output_dir = os.path.join(self.system.cv_HOST.results_dir, output_dir, "hmc-" + hmc.hmc_ip) if not os.path.exists(output_dir): os.makedirs(output_dir) default_commands = ['lshmc -V', 'pedbg -c -q 4'] list_of_commands.extend(default_commands) try: for cmd in set(list_of_commands): output = "\n".join(hmc.run_command(r"%s" % cmd, timeout=1800)) filename = "%s.log" % '-'.join( (re.sub(r'[^a-zA-Z0-9]', ' ', cmd)).split()) filepath = os.path.join(output_dir, filename) with open(filepath, 'w') as f: f.write(output) log.warn( "Please collect the pedbg logs immediately. At risk of being overwritten." ) return True except CommandFailed as cmd_failed: raise cmd_failed def run_command_ignore_fail(self, command, timeout=60, retry=0): ''' Wrapper function for `ssh.run_command_ignore_fail` :param command: string, command :param timeout: number, time out in seconds :param retry: number, number of retries ''' return self.ssh.run_command_ignore_fail(command, timeout * self.timeout_factor, retry) def run_command(self, i_cmd, timeout=15): ''' Wrapper function for `ssh.run_command` :param i_cmd: string, command :param timeout: number, time out in seconds ''' return self.ssh.run_command(i_cmd, timeout)