class ControlC(unittest.TestCase):
'''
Start a process that does a bunch of console output, and then try and
'control-c' it to stop the process and get a prompt back.
'''
CONTROL = 'c'
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_BMC = conf.bmc()
self.cv_SYSTEM = conf.system()
self.util = OpTestUtil()
self.prompt = self.util.build_prompt()
def cleanup(self):
pass
def runTest(self):
self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)
console = self.cv_BMC.get_host_console()
if (isinstance(self.cv_BMC, OpTestMambo.OpTestMambo)):
raise unittest.SkipTest("Mambo so skipping Control-C tests")
raw_pty = console.get_console()
raw_pty.sendline("find /")
time.sleep(2)
raw_pty.sendcontrol(self.CONTROL)
BMC_DISCONNECT = 'SOL session closed by BMC'
timeout = 60
try:
rc = raw_pty.expect([BMC_DISCONNECT, self.prompt,
pexpect.TIMEOUT, pexpect.EOF], timeout)
if rc == 0:
raise BMCDisconnected(BMC_DISCONNECT)
self.assertEqual(rc, 1, "Failed to find expected prompt")
except pexpect.TIMEOUT as e:
log.debug(e)
log.debug("# TIMEOUT waiting for command to finish with ctrl-c.")
log.debug("# Everything is terrible. Fail the world, "
"power cycle (if lucky)")
self.cv_SYSTEM.set_state(OpSystemState.UNKNOWN_BAD)
self.fail("Could not ctrl-c running command in reasonable time")
self.cleanup()
class ControlC(unittest.TestCase):
CONTROL = 'c'
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_BMC = conf.bmc()
self.cv_SYSTEM = conf.system()
self.util = OpTestUtil()
self.prompt = self.util.build_prompt()
def cleanup(self):
pass
def runTest(self):
self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)
console = self.cv_BMC.get_host_console()
# I should really make this API less nasty...
raw_console = console.get_console()
#raw_console.sendline("hexdump -C -v /dev/zero")
raw_console.sendline("find /")
time.sleep(2)
raw_console.sendcontrol(self.CONTROL)
BMC_DISCONNECT = 'SOL session closed by BMC'
timeout = 60
try:
rc = raw_console.expect(
[BMC_DISCONNECT, self.prompt, pexpect.TIMEOUT, pexpect.EOF],
timeout)
if rc == 0:
raise BMCDisconnected(BMC_DISCONNECT)
self.assertEqual(rc, 1, "Failed to find expected prompt")
except pexpect.TIMEOUT as e:
log.debug(e)
log.debug("# TIMEOUT waiting for command to finish with ctrl-c.")
log.debug(
"# Everything is terrible. Fail the world, power cycle (if lucky)"
)
self.cv_SYSTEM.set_state(OpSystemState.UNKNOWN_BAD)
self.fail("Could not ctrl-c running command in reasonable time")
self.cleanup()
class HMCConsole(HMCUtil):
"""
HMCConsole Class
Methods to manage the console of LPAR
"""
def __init__(self,
hmc_ip,
user_name,
password,
managed_system,
lpar_name,
lpar_vios,
lpar_prof,
lpar_user,
lpar_password,
block_setup_term=None,
delaybeforesend=None,
timeout_factor=1,
logfile=sys.stdout,
prompt=None,
scratch_disk="",
check_ssh_keys=False,
known_hosts_file=None,
proxy=""):
self.logfile = logfile
self.hmc_ip = hmc_ip
self.user = user_name
self.passwd = password
self.mg_system = managed_system
self.util = OpTestUtil()
self.expect_prompt = self.util.build_prompt(prompt) + "$"
self.lpar_name = lpar_name
self.lpar_vios = lpar_vios
self.lpar_prof = lpar_prof
self.lpar_user = lpar_user
self.lpar_password = lpar_password
self.scratch_disk = scratch_disk
self.proxy = proxy
self.state = ConsoleState.DISCONNECTED
self.delaybeforesend = delaybeforesend
self.system = None
# OpTestUtil instance is NOT conf's
self.prompt = prompt
self.pty = None
self.delaybeforesend = delaybeforesend
# allows caller specific control of when to block setup_term
self.block_setup_term = block_setup_term
# tells setup_term to not throw exceptions, like when system off
self.setup_term_quiet = 0
# flags the object to abandon setup_term operations, like when system off
self.setup_term_disable = 0
# FUTURE - System Console currently tracked in System Object
# state tracking, reset on boot and state changes
self.PS1_set = -1
self.LOGIN_set = -1
self.SUDO_set = -1
super(HMCConsole,
self).__init__(hmc_ip, user_name, password, scratch_disk, proxy,
logfile, managed_system, lpar_name, prompt,
block_setup_term, delaybeforesend, timeout_factor,
lpar_prof, lpar_vios, lpar_user, lpar_password,
check_ssh_keys, known_hosts_file)
def set_system(self, system):
self.ssh.set_system(system)
self.system = system
self.pty = self.get_console()
self.pty.set_system(system)
def get_host_console(self):
return self.pty
def set_system_setup_term(self, flag):
self.system.block_setup_term = flag
def get_system_setup_term(self):
return self.system.block_setup_term
def get_scratch_disk(self):
return self.scratch_disk
def get_proxy(self):
return self.proxy
def hostname(self):
return self.hmc_ip
def username(self):
return self.user
def password(self):
return self.passwd
def set_block_setup_term(self, flag):
self.block_setup_term = flag
def get_block_setup_term(self):
return self.block_setup_term
def enable_setup_term_quiet(self):
self.setup_term_quiet = 1
self.setup_term_disable = 0
def disable_setup_term_quiet(self):
self.setup_term_quiet = 0
self.setup_term_disable = 0
def close(self):
self.util.clear_state(self)
try:
self.pty.close()
if self.pty.status != -1: # leaving for debug
if os.WIFEXITED(self.pty.status):
os.WEXITSTATUS(self.pty.status)
else:
os.WTERMSIG(self.pty.status)
self.state = ConsoleState.DISCONNECTED
except pexpect.ExceptionPexpect:
self.state = ConsoleState.DISCONNECTED
raise "HMC Console: failed to close console"
except Exception:
self.state = ConsoleState.DISCONNECTED
log.debug("HMC close -> TERMINATE")
def connect(self, logger=None):
if self.state == ConsoleState.CONNECTED:
return self.pty
self.util.clear_state(self) # clear when coming in DISCONNECTED
log.info("De-activating the console")
self.deactivate_lpar_console()
log.debug("#HMC Console CONNECT")
command = "sshpass -p %s ssh -p 22 -l %s %s -o PubkeyAuthentication=no"\
" -o afstokenpassing=no -q -o 'UserKnownHostsFile=/dev/null'"\
" -o 'StrictHostKeyChecking=no'"
try:
self.pty = Spawn(command % (self.passwd, self.user, self.hmc_ip))
log.info("Opening the LPAR console")
time.sleep(STALLTIME)
self.pty.send('\r')
self.pty.sendline("mkvterm -m %s -p %s" %
(self.mg_system, self.lpar_name))
self.pty.send('\r')
time.sleep(STALLTIME)
i = self.pty.expect(["Open Completed.", pexpect.TIMEOUT],
timeout=60)
self.pty.logfile = sys.stdout
if logger:
self.pty.logfile_read = OpTestLogger.FileLikeLogger(logger)
else:
self.pty.logfile_read = OpTestLogger.FileLikeLogger(log)
if i == 0:
time.sleep(STALLTIME)
self.state = ConsoleState.CONNECTED
self.pty.setwinsize(1000, 1000)
else:
raise OpTestError("Check the lpar activate command")
except Exception as exp:
self.state = ConsoleState.DISCONNECTED
raise CommandFailed(
'OPexpect.spawn',
'OPexpect.spawn encountered a problem: ' + str(exp), -1)
if self.delaybeforesend:
self.pty.delaybeforesend = self.delaybeforesend
if not self.pty.isalive():
raise CommandFailed("mkvterm", self.pty.read(), self.pty.status)
return self.pty
def check_state(self):
return self.state
def get_console(self, logger=None):
if self.state == ConsoleState.DISCONNECTED:
self.util.clear_state(self)
self.connect(logger=logger)
time.sleep(STALLTIME)
l_rc = self.pty.expect(["login:"******"Shell prompt changed")
else:
self.pty.send('\r')
log.debug("Waiting till booting!")
self.pty = self.get_login_prompt()
if self.system.SUDO_set != 1 or self.system.LOGIN_set != 1 or self.system.PS1_set != 1:
self.util.setup_term(self.system, self.pty, None,
self.system.block_setup_term)
# Clear buffer before usage
self.pty.buffer = ""
return self.pty
def get_login_prompt(self):
# Assuming 'Normal' boot set in LPAR profile
# We wait for upto 500 seconds for LPAR to boot to OS
self.pty.send('\r')
time.sleep(STALLTIME)
log.debug("Waiting for login screen")
i = self.pty.expect(["login:"******"System has booted")
time.sleep(STALLTIME)
elif i == 1:
log.debug("Console already logged in")
else:
log.error("Failed to get login prompt %s", self.pty.before)
# To cheat system for making using of HMC SSH
self.system.PS1_set = 1
self.system.LOGIN_set = 1
self.system.SUDO_set = 1
return self.pty
def run_command(self, i_cmd, timeout=15):
return self.util.run_command(self, i_cmd, timeout)
class HMCUtil():
'''
Utility and functions of HMC object
'''
def __init__(self,
hmc_ip,
user_name,
password,
scratch_disk="",
proxy="",
logfile=sys.stdout,
managed_system=None,
lpar_name=None,
prompt=None,
block_setup_term=None,
delaybeforesend=None,
timeout_factor=None,
lpar_prof=None,
lpar_vios=None,
lpar_user=None,
lpar_password=None,
check_ssh_keys=False,
known_hosts_file=None,
tgt_managed_system=None):
self.hmc_ip = hmc_ip
self.user = user_name
self.passwd = password
self.logfile = logfile
self.mg_system = managed_system
self.tgt_mg_system = tgt_managed_system
self.check_ssh_keys = check_ssh_keys
self.known_hosts_file = known_hosts_file
self.lpar_name = lpar_name
self.lpar_prof = lpar_prof
self.lpar_user = lpar_user
self.lpar_password = lpar_password
self.lpar_vios = lpar_vios
self.util = OpTestUtil()
self.prompt = prompt
self.expect_prompt = self.util.build_prompt(prompt) + "$"
self.ssh = OpTestSSH(hmc_ip,
user_name,
password,
logfile=self.logfile,
check_ssh_keys=check_ssh_keys,
known_hosts_file=known_hosts_file,
block_setup_term=block_setup_term)
self.scratch_disk = scratch_disk
self.proxy = proxy
self.scratch_disk_size = None
self.delaybeforesend = delaybeforesend
self.system = None
# OpTestUtil instance is NOT conf's
self.pty = None
# allows caller specific control of when to block setup_term
self.block_setup_term = block_setup_term
# tells setup_term to not throw exceptions, like when system off
self.setup_term_quiet = 0
# flags the object to abandon setup_term operations, like when system off
self.setup_term_disable = 0
# functional simulators are very slow, so multiply all default timeouts by this factor
self.timeout_factor = timeout_factor
# state tracking, reset on boot and state changes
# console tracking done on System object for the system console
self.PS1_set = -1
self.LOGIN_set = -1
self.SUDO_set = -1
def deactivate_lpar_console(self):
self.ssh.run_command("rmvterm -m %s -p %s" %
(self.mg_system, self.lpar_name),
timeout=10)
def poweroff_system(self):
if self.get_system_state() != OpManagedState.OPERATING:
raise OpTestError('Managed Systen not in Operating state')
self.ssh.run_command("chsysstate -m %s -r sys -o off" % self.mg_system)
self.wait_system_state(OpManagedState.OFF)
def poweron_system(self):
if self.get_system_state() != OpManagedState.OFF:
raise OpTestError('Managed Systen not is Power off state!')
self.ssh.run_command("chsysstate -m %s -r sys -o on" % self.mg_system)
self.wait_system_state()
if self.lpar_vios:
log.debug("Starting VIOS %s", self.lpar_vios)
self.poweron_lpar(vios=True)
def poweroff_lpar(self):
if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
log.info('LPAR Already powered-off!')
return
self.ssh.run_command(
"chsysstate -m %s -r lpar -n %s -o shutdown --immed" %
(self.mg_system, self.lpar_name))
self.wait_lpar_state(OpHmcState.NOT_ACTIVE)
def poweron_lpar(self, vios=False):
if self.get_lpar_state(vios) == OpHmcState.RUNNING:
log.info('LPAR Already powered on!')
return BMC_CONST.FW_SUCCESS
lpar_name = self.lpar_name
if vios:
lpar_name = self.lpar_vios
cmd = "chsysstate -m %s -r lpar -n %s -o on" % (self.mg_system,
lpar_name)
if self.lpar_prof:
cmd = "%s -f %s" % (cmd, self.lpar_prof)
self.wait_lpar_state(OpHmcState.NOT_ACTIVE, vios=vios)
self.ssh.run_command(cmd)
self.wait_lpar_state(vios=vios)
time.sleep(STALLTIME)
return BMC_CONST.FW_SUCCESS
def dumprestart_lpar(self):
if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
log.info('LPAR Already powered-off!')
return
self.ssh.run_command("chsysstate -m %s -r lpar -n %s -o dumprestart" %
(self.mg_system, self.lpar_name))
self.wait_lpar_state()
def restart_lpar(self):
if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
log.info('LPAR Already powered-off!')
return
self.ssh.run_command(
"chsysstate -m %s -r lpar -n %s -o shutdown --immed --restart" %
(self.mg_system, self.lpar_name))
self.wait_lpar_state()
def get_lpar_state(self, vios=False):
lpar_name = self.lpar_name
if vios:
lpar_name = self.lpar_vios
state = self.ssh.run_command(
'lssyscfg -m %s -r lpar --filter lpar_names=%s -F state' %
(self.mg_system, lpar_name))[-1]
ref_code = self.ssh.run_command(
'lsrefcode -m %s -r lpar --filter lpar_names=%s -F refcode' %
(self.mg_system, lpar_name))[-1]
if state == 'Running':
if 'Linux' in ref_code or not ref_code:
return 'Running'
else:
return 'Booting'
return state
def get_system_state(self):
state = self.ssh.run_command('lssyscfg -m %s -r sys -F state' %
self.mg_system)
return state[-1]
def wait_lpar_state(self,
exp_state=OpHmcState.RUNNING,
vios=False,
timeout=WAITTIME):
state = self.get_lpar_state(vios)
count = 0
while state != exp_state:
state = self.get_lpar_state(vios)
log.info("Current state: %s", state)
time.sleep(timeout)
count = 1
if count > 120:
raise OpTestError("Time exceeded for reaching %s" % exp_state)
def wait_system_state(self,
exp_state=OpManagedState.OPERATING,
timeout=WAITTIME):
state = self.get_system_state()
count = 0
while state != exp_state:
state = self.get_system_state()
log.info("Current state: %s", state)
time.sleep(timeout)
count = 1
if count > 60:
raise OpTestError("Time exceeded for reaching %s" % exp_state)
def is_lpar_in_managed_system(self, mg_system=None):
lpar_list = self.ssh.run_command('lssyscfg -r lpar -m %s -F name' %
mg_system)
if self.lpar_name in lpar_list:
log.info("%s lpar found in managed system %s" %
(mg_system, self.lpar_name))
return True
return False
def migrate_lpar(self, src_mg_system=None, dest_mg_system=None):
if src_mg_system == None or dest_mg_system == None:
raise OpTestError(
"Source and Destination Managed System required for LPM")
if not self.is_lpar_in_managed_system(src_mg_system):
raise OpTestError("Lpar %s not found in managed system %s" %
(self.lpar_name, src_mg_system))
self.ssh.run_command('migrlpar -o v -m %s -t %s -p %s' %
(src_mg_system, dest_mg_system, self.lpar_name))
self.ssh.run_command('migrlpar -o m -m %s -t %s -p %s' %
(src_mg_system, dest_mg_system, self.lpar_name))
if self.is_lpar_in_managed_system(dest_mg_system):
log.info("Migration of lpar %s from %s to %s is successfull" %
(self.lpar_name, src_mg_system, dest_mg_system))
self.mg_system = dest_mg_system
return True
log.info("Migration of lpar %s from %s to %s failed" %
(self.lpar_name, src_mg_system, dest_mg_system))
return False
def run_command_ignore_fail(self, command, timeout=60, retry=0):
return self.ssh.run_command_ignore_fail(command,
timeout * self.timeout_factor,
retry)
def run_command(self, i_cmd, timeout=15):
return self.ssh.run_command(i_cmd, timeout)
class OpTestHMC():
'''
This class contains the modules to perform various HMC operations on an LPAR.
The Host IP, username and password of HMC have to be passed to the class intially
while creating the object for the class.
'''
def __init__(self,
hmc_ip,
user_name,
password,
scratch_disk="",
proxy="",
logfile=sys.stdout,
managed_system=None,
lpar_name=None,
prompt=None,
lpar_prof=None,
lpar_vios=None,
lpar_user=None,
lpar_password=None,
check_ssh_keys=False,
known_hosts_file=None):
self.hmc_ip = hmc_ip
self.user = user_name
self.passwd = password
self.logfile = logfile
self.system = managed_system
self.system = managed_system
self.check_ssh_keys = check_ssh_keys
self.known_hosts_file = known_hosts_file
self.lpar_name = lpar_name
self.lpar_prof = lpar_prof
self.lpar_user = lpar_user
self.lpar_password = lpar_password
self.lpar_vios = lpar_vios
self.lpar_con = None
self.vterm = False
self.util = OpTestUtil()
self.prompt = prompt
self.expect_prompt = self.util.build_prompt(prompt) + "$"
self.conf = OpTestConfiguration.conf
self.ssh = OpTestSSH(hmc_ip,
user_name,
password,
logfile=self.logfile,
check_ssh_keys=check_ssh_keys,
known_hosts_file=known_hosts_file)
self.scratch_disk = scratch_disk
self.proxy = proxy
self.scratch_disk_size = None
def hostname(self):
return self.hmc_ip
def username(self):
return self.user
def password(self):
return self.passwd
def set_system(self, system):
self.ssh.set_system(system)
def get_scratch_disk(self):
return self.scratch_disk
def run_command(self, i_cmd, timeout=15):
return self.ssh.run_command(i_cmd, timeout)
def deactivate_lpar_console(self):
self.run_command("rmvterm -m %s -p %s" % (self.system, self.lpar_name),
timeout=10)
def poweroff_system(self):
if self.get_system_state() != OpManagedState.OPERATING:
raise OpTestError('Managed Systen not in Operating state')
self.run_command("chsysstate -m %s -r sys -o off" % self.system)
self.wait_system_state(OpManagedState.OFF)
def poweron_system(self):
if self.get_system_state() != OpManagedState.OFF:
raise OpTestError('Managed Systen not is Power off state!')
self.run_command("chsysstate -m %s -r sys -o on" % self.system)
self.wait_system_state()
if self.lpar_vios:
log.debug("Starting VIOS %s" % self.lpar_vios)
self.poweron_lpar(vios=True)
def poweroff_lpar(self):
if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
log.info('LPAR Already powered-off!')
return
self.run_command("chsysstate -m %s -r lpar -n %s -o shutdown --immed" %
(self.system, self.lpar_name))
self.wait_lpar_state(OpHmcState.NOT_ACTIVE)
def poweron_lpar(self, runtime=False, vios=False):
if self.get_lpar_state(vios) == OpHmcState.RUNNING:
log.info('LPAR Already powered on!')
return
lpar_name = self.lpar_name
if vios:
lpar_name = self.lpar_vios
cmd = "chsysstate -m %s -r lpar -n %s -o on" % (self.system, lpar_name)
if not vios:
if self.lpar_prof:
cmd = "%s -f %s" % (cmd, self.lpar_prof)
self.wait_lpar_state(OpHmcState.NOT_ACTIVE, vios=vios)
self.run_command(cmd)
self.wait_lpar_state(vios=vios)
if runtime:
self.wait_login_prompt(self.get_console_prompt())
self.close_console(self.lpar_con)
def get_lpar_state(self, vios=False):
lpar_name = self.lpar_name
if vios:
lpar_name = self.lpar_vios
state = self.run_command(
'lssyscfg -m %s -r lpar --filter lpar_names=%s -F state' %
(self.system, lpar_name))
return state[-1]
def get_system_state(self):
state = self.run_command('lssyscfg -m %s -r sys -F state' %
self.system)
return state[-1]
def wait_lpar_state(self,
exp_state=OpHmcState.RUNNING,
vios=False,
timeout=WAITTIME):
state = self.get_lpar_state(vios)
count = 0
while state != exp_state:
state = self.get_lpar_state(vios)
log.info("Current state: %s" % state)
time.sleep(timeout)
count = 1
if count > 120:
raise OpTestError("Time exceeded for reaching %s" % exp_state)
def wait_system_state(self,
exp_state=OpManagedState.OPERATING,
timeout=WAITTIME):
state = self.get_system_state()
count = 0
while state != exp_state:
state = self.get_system_state()
log.info("Current state: %s" % state)
time.sleep(timeout)
count = 1
if count > 60:
raise OpTestError("Time exceeded for reaching %s" % exp_state)
def vterm_run_command(self, console, cmd, timeout=60):
if console is None:
console = self.get_console()
try:
console.send(cmd)
console.send('\r')
output = console.before
console.sendline("echo $?")
console.send('\r')
# Time to accumulate both outputs
time.sleep(STALLTIME)
rc = console.expect([self.expect_prompt], timeout=timeout)
if rc != 0:
exitcode = int(console.before)
log.debug("# LAST COMMAND EXIT CODE %d (%s)" %
(exitcode, repr(console.before)))
except pexpect.TIMEOUT as e:
log.debug(e)
log.debug("# TIMEOUT waiting for command to finish.")
log.debug("# Attempting to control-c")
try:
console.sendcontrol('c')
rc = console.expect([self.expect_prompt], 10)
if rc == 0:
raise CommandFailed(cmd, "TIMEOUT", -1)
except pexpect.TIMEOUT:
log.error("# Timeout trying to kill timed-out command.")
log.error(
"# Failing current command and attempting to continue")
self.deactivate_lpar_console()
raise CommandFailed("console", "timeout", -1)
raise e
if rc == 0:
res = console.before
res = res.split(cmd)
return res[-1].splitlines()[1:-2]
else:
res = output
res = res.splitlines()
if exitcode != 0:
raise CommandFailed(cmd, res, exitcode)
return res
def get_console(self):
console = self.get_console_prompt()
time.sleep(STALLTIME)
l_rc = console.expect(["login:"******"Shell prompt changed")
else:
console.send('\r')
log.debug("Waiting till booting!")
console = self.wait_login_prompt(console, self.lpar_user,
self.lpar_password)
return console
def check_vterm(self):
return self.vterm
def wait_login_prompt(self, console, username=None, password=None):
if not console:
raise OpTestError("Console is not provided")
# Assuming 'Normal' boot set in LPAR profile
# We wait for upto 500 seconds for LPAR to boot to OS
console.buffer = ""
console.send('\r')
time.sleep(STALLTIME)
log.debug("Waiting for login screen")
i = console.expect(["login:"******"System has booted")
if username and password:
self.lpar_con.sendline(username)
self.lpar_con.send('\r')
time.sleep(STALLTIME)
i = self.lpar_con.expect(["Password:"******"Last login", "incorrect", pexpect.TIMEOUT],
timeout=60)
if i == 0:
log.info('Logged in to console')
time.sleep(STALLTIME)
return self.lpar_con
elif i == 1:
raise OpTestError("Wrong Credentials for host")
else:
raise OpTestError("Upexpected return")
time.sleep(STALLTIME)
return self.lpar_con
elif i == 1:
# Assuming console already logged in !"
return self.lpar_con
else:
log.error("%s %s" % (i, self.lpar_con.before))
raise OpTestError("Console in different state")
def get_console_prompt(self):
if self.get_lpar_state() != OpHmcState.RUNNING:
raise OpTestError('LPAR is not in Running State. Please check!')
log.info("De-activating the console")
self.deactivate_lpar_console()
command = "sshpass -p %s ssh -p 22 -l %s %s -o PubkeyAuthentication=no"\
" -o afstokenpassing=no -q -o 'UserKnownHostsFile=/dev/null'"\
" -o 'StrictHostKeyChecking=no'"
self.lpar_con = OPexpect.spawn(command %
(self.passwd, self.user, self.hmc_ip))
self.wait_lpar_state()
log.info("Opening the LPAR console")
self.vterm = True
self.lpar_con.sendline("mkvterm -m %s -p %s" %
(self.system, self.lpar_name))
self.lpar_con.send('\r')
time.sleep(STALLTIME)
i = self.lpar_con.expect(["Open Completed.", pexpect.TIMEOUT],
timeout=60)
self.lpar_con.logfile = sys.stdout
self.lpar_con.logfile_read = OpTestLogger.FileLikeLogger(log)
if i == 0:
time.sleep(STALLTIME)
return self.lpar_con
else:
raise OpTestError("Check the lpar activate command")
def close_console(self, console):
console.send('~.')
self.vterm = False
time.sleep(STALLTIME)
console.close()
class HMCUtil():
'''
Utility and functions of HMC object
'''
def __init__(self,
hmc_ip,
user_name,
password,
scratch_disk="",
proxy="",
logfile=sys.stdout,
managed_system=None,
lpar_name=None,
prompt=None,
block_setup_term=None,
delaybeforesend=None,
timeout_factor=None,
lpar_prof=None,
lpar_vios=None,
lpar_user=None,
lpar_password=None,
check_ssh_keys=False,
known_hosts_file=None,
tgt_managed_system=None,
tgt_lpar=None):
self.hmc_ip = hmc_ip
self.user = user_name
self.passwd = password
self.logfile = logfile
self.mg_system = managed_system
self.tgt_mg_system = tgt_managed_system
self.tgt_lpar = tgt_lpar
self.check_ssh_keys = check_ssh_keys
self.known_hosts_file = known_hosts_file
self.lpar_name = lpar_name
self.lpar_prof = lpar_prof
self.lpar_user = lpar_user
self.lpar_password = lpar_password
self.lpar_vios = lpar_vios
self.util = OpTestUtil()
self.prompt = prompt
self.expect_prompt = self.util.build_prompt(prompt) + "$"
self.ssh = OpTestSSH(hmc_ip,
user_name,
password,
logfile=self.logfile,
check_ssh_keys=check_ssh_keys,
known_hosts_file=known_hosts_file,
block_setup_term=block_setup_term)
self.scratch_disk = scratch_disk
self.proxy = proxy
self.scratch_disk_size = None
self.delaybeforesend = delaybeforesend
self.system = None
# OpTestUtil instance is NOT conf's
self.pty = None
# allows caller specific control of when to block setup_term
self.block_setup_term = block_setup_term
# tells setup_term to not throw exceptions, like when system off
self.setup_term_quiet = 0
# flags the object to abandon setup_term operations, like when system off
self.setup_term_disable = 0
# functional simulators are very slow, so multiply all default timeouts by this factor
self.timeout_factor = timeout_factor
# state tracking, reset on boot and state changes
# console tracking done on System object for the system console
self.PS1_set = -1
self.LOGIN_set = -1
self.SUDO_set = -1
def deactivate_lpar_console(self):
'''
Deactivate/disconnect the LPAR Console
'''
self.ssh.run_command("rmvterm -m %s -p %s" %
(self.mg_system, self.lpar_name),
timeout=10)
def poweroff_system(self):
'''
PowerOFF the managed system
'''
if self.get_system_state() != OpManagedState.OPERATING:
raise OpTestError('Managed Systen not in Operating state')
self.ssh.run_command("chsysstate -m %s -r sys -o off" % self.mg_system)
self.wait_system_state(OpManagedState.OFF)
def poweron_system(self):
'''
PowerON the managed system
'''
if self.get_system_state() != OpManagedState.OFF:
raise OpTestError('Managed Systen not is Power off state!')
self.ssh.run_command("chsysstate -m %s -r sys -o on" % self.mg_system)
self.wait_system_state()
if self.lpar_vios:
log.debug("Starting VIOS %s", self.lpar_vios)
self.poweron_lpar(vios=True)
def poweroff_lpar(self, remote_hmc=None):
'''
PowerOFF the LPAR
:param remote_hmc: object, remote HMC instance
'''
hmc = remote_hmc if remote_hmc else self
if self.get_lpar_state(remote_hmc=remote_hmc) in [
OpHmcState.NOT_ACTIVE, OpHmcState.NA
]:
log.info('LPAR Already powered-off!')
return
hmc.ssh.run_command(
"chsysstate -m %s -r lpar -n %s -o shutdown --immed" %
(hmc.mg_system, self.lpar_name))
self.wait_lpar_state(OpHmcState.NOT_ACTIVE, remote_hmc=remote_hmc)
def poweron_lpar(self, vios=False, remote_hmc=None):
'''
PowerON the LPAR
:param vios: Boolean, to identify VIOS partition
:param remote_hmc: object, remote HMC instance
:returns: BMC_CONST.FW_SUCCESS up on success
'''
hmc = remote_hmc if remote_hmc else self
if self.get_lpar_state(vios,
remote_hmc=remote_hmc) == OpHmcState.RUNNING:
log.info('LPAR Already powered on!')
return BMC_CONST.FW_SUCCESS
lpar_name = self.lpar_name
if vios:
lpar_name = hmc.lpar_vios
cmd = "chsysstate -m %s -r lpar -n %s -o on" % (hmc.mg_system,
lpar_name)
if self.lpar_prof:
cmd = "%s -f %s" % (cmd, self.lpar_prof)
self.wait_lpar_state(OpHmcState.NOT_ACTIVE,
vios=vios,
remote_hmc=remote_hmc)
hmc.ssh.run_command(cmd)
self.wait_lpar_state(vios=vios, remote_hmc=remote_hmc)
time.sleep(STALLTIME)
return BMC_CONST.FW_SUCCESS
def dumprestart_lpar(self):
'''
Capture dump and restart the LPAR
'''
if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
log.info('LPAR Already powered-off!')
return
self.ssh.run_command("chsysstate -m %s -r lpar -n %s -o dumprestart" %
(self.mg_system, self.lpar_name))
self.wait_lpar_state()
def restart_lpar(self):
'''
Restart LPAR immediately
'''
if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
log.info('LPAR Already powered-off!')
return
self.ssh.run_command(
"chsysstate -m %s -r lpar -n %s -o shutdown --immed --restart" %
(self.mg_system, self.lpar_name))
self.wait_lpar_state()
def get_lpar_cfg(self):
'''
Get LPAR configuration parameters
:returns: LPAR configuration parameters in key, value pair
'''
out = self.ssh.run_command(
"lssyscfg -r prof -m %s --filter 'lpar_names=%s'" %
(self.mg_system, self.lpar_name))[-1]
cfg_dict = {}
splitter = shlex.shlex(out)
splitter.whitespace += ','
splitter.whitespace_split = True
for values in list(splitter):
data = values.split("=")
key = data[0]
value = data[1]
cfg_dict[key] = value
return cfg_dict
def set_lpar_cfg(self, arg_str):
'''
Set LPAR configuration parameter values
:param arg_str: configuration values in key, value pair seperated by
comma
'''
if not self.lpar_prof:
raise OpTestError("Profile needs to be defined to use this method")
self.ssh.run_command(
"chsyscfg -r prof -m %s -p %s -i 'lpar_name=%s,name=%s,%s' --force"
% (self.mg_system, self.lpar_name, self.lpar_name, self.lpar_prof,
arg_str))
def get_lpar_state(self, vios=False, remote_hmc=None):
'''
Get current state of LPAR
:param vios: Boolean, to identify VIOS partition
:param remote_hmc: object, remote HMC instance
:returns: the current status of the LPAR e.g 'Running' or 'Booting'
'''
hmc = remote_hmc if remote_hmc else self
lpar_name = self.lpar_name
if vios:
lpar_name = hmc.lpar_vios
state = hmc.ssh.run_command(
'lssyscfg -m %s -r lpar --filter lpar_names=%s -F state' %
(hmc.mg_system, lpar_name))[-1]
ref_code = hmc.ssh.run_command(
'lsrefcode -m %s -r lpar --filter lpar_names=%s -F refcode' %
(hmc.mg_system, lpar_name))[-1]
if state == 'Running':
if 'Linux' in ref_code or not ref_code:
return 'Running'
else:
return 'Booting'
return state
def get_system_state(self):
'''
Get current state of managed system
:returns: the current status of the managed system
'''
state = self.ssh.run_command('lssyscfg -m %s -r sys -F state' %
self.mg_system)
return state[-1]
def wait_lpar_state(self,
exp_state=OpHmcState.RUNNING,
vios=False,
timeout=WAITTIME,
remote_hmc=None):
'''
Wait for a particular state of LPAR
:param exp_state: constant, expected HMC STATE, default is OpHmcState.RUNNING
:param vios: Boolean, to identify VIOS partition
:param timeout: number, Wait time in seconds
:param remote_hmc: object, remote HMC instance
:raises: :class:`common.OpTestError` when the timeout happens
'''
state = self.get_lpar_state(vios, remote_hmc=remote_hmc)
count = 0
while state != exp_state:
state = self.get_lpar_state(vios, remote_hmc=remote_hmc)
log.info("Current state: %s", state)
time.sleep(timeout)
count += 1
if count > 120:
raise OpTestError("Time exceeded for reaching %s" % exp_state)
def wait_system_state(self,
exp_state=OpManagedState.OPERATING,
timeout=SYS_WAITTIME):
'''
Wait for a particular state of managed system
:param exp_state: constatn, expected STATE, default is OpManagedState.OPERATING
:param timeout: number, Wait time in seconds
:raises: :class:`common.OpTestError` when the timeout happens
'''
state = self.get_system_state()
count = 0
while state != exp_state:
state = self.get_system_state()
log.info("Current state: %s", state)
time.sleep(timeout)
count += 1
if count > 60:
raise OpTestError("Time exceeded for reaching %s" % exp_state)
def is_lpar_in_managed_system(self,
mg_system=None,
lpar_name=None,
remote_hmc=None):
'''
Check if LPAR is available on a managed system
:param mg_system: string, managed system name
:param lpar_name: string, LPAR name
:param remote_hmc: object, remote HMC instance
:returns: True - when the LPAR is in managed system
False - when the LPAR is not in managed system
'''
hmc = remote_hmc if remote_hmc else self
lpar_list = hmc.ssh.run_command('lssyscfg -r lpar -m %s -F name' %
mg_system)
if lpar_name in lpar_list:
log.info("%s lpar found in managed system %s" %
(lpar_name, mg_system))
return True
return False
def migrate_lpar(self,
src_mg_system=None,
dest_mg_system=None,
options=None,
param="",
timeout=300):
'''
Migrate the LPAR from source managed system to the destination managed
system
:param src_mg_system: string, source managed system name
:param dest_mg_system: string, destination manages system name
:param options: string, options for migration
:param param: string, required parameters for migration
:param timeout: number, time out value in seconds, default 300
:returns: True - when LPAR migration success
False - when LPAR migration unsuccess
'''
if src_mg_system == None or dest_mg_system == None:
raise OpTestError(
"Source and Destination Managed System required for LPM")
if not self.is_lpar_in_managed_system(src_mg_system, self.lpar_name):
raise OpTestError("Lpar %s not found in managed system %s" %
(self.lpar_name, src_mg_system))
for mode in ['v', 'm']:
cmd = 'migrlpar -o %s -m %s -t %s -p %s %s' % (
mode, src_mg_system, dest_mg_system, self.lpar_name, param)
if options:
cmd = "%s %s" % (cmd, options)
self.ssh.run_command(cmd, timeout=timeout)
log.debug("Waiting for %.2f minutes." % (timeout / 60))
time.sleep(timeout)
if self.is_lpar_in_managed_system(dest_mg_system, self.lpar_name):
cmd = "lssyscfg -m %s -r lpar --filter lpar_names=%s -F state" % (
dest_mg_system, self.lpar_name)
lpar_state = self.ssh.run_command(cmd)[0]
if lpar_state not in [
'Migrating - Running', 'Migrating - Not Activated'
]:
log.info("Migration of lpar %s from %s to %s is successfull" %
(self.lpar_name, src_mg_system, dest_mg_system))
self.mg_system = dest_mg_system
return True
self.recover_lpar(src_mg_system,
dest_mg_system,
stop_lpm=True,
timeout=timeout)
log.info("Migration of lpar %s from %s to %s failed" %
(self.lpar_name, src_mg_system, dest_mg_system))
return False
def set_ssh_key_auth(self, hmc_ip, hmc_user, hmc_passwd, remote_hmc=None):
'''
Set SSH authentication keys
:param hmc_ip: number, in ip address format
:param hmc_user: string, HMC user name
:param hmc_passwd: string, HMC user password
:param remote_hmc: object, remote HMC instance
:raises: `CommandFailed`
'''
hmc = remote_hmc if remote_hmc else self
try:
cmd = "mkauthkeys -u %s --ip %s --test" % (hmc_user, hmc_ip)
hmc.run_command(cmd, timeout=120)
except CommandFailed:
try:
cmd = "mkauthkeys -u %s --ip %s --passwd %s" % (
hmc_user, hmc_ip, hmc_passwd)
hmc.run_command(cmd, timeout=120)
except CommandFailed as cf:
raise cf
def cross_hmc_migration(self,
src_mg_system=None,
dest_mg_system=None,
target_hmc_ip=None,
target_hmc_user=None,
target_hmc_passwd=None,
remote_hmc=None,
options=None,
param="",
timeout=300):
'''
migration of LPAR across HMC
:param src_mg_system: string, Source managed system
:param dest_mg_system: string, Destination managed system
:param target_hmc_ip: number, in IP address format
:param target_hmc_user: string, target HMC user name
:param target_hmc_passwd: string, target HMC user password
:param remote_hmc: object, remote HMC instance
:param options: string, migration options
:param param: string, migration parameters
:param timeout: number, time in seconds
'''
hmc = remote_hmc if remote_hmc else self
if src_mg_system == None or dest_mg_system == None:
raise OpTestError("Source and Destination Managed System "\
"required for Cross HMC LPM")
if target_hmc_ip == None or target_hmc_user == None or target_hmc_passwd == None:
raise OpTestError("Destination HMC IP, Username, and Password "\
"required for Cross HMC LPM")
if not self.is_lpar_in_managed_system(src_mg_system, self.lpar_name,
remote_hmc):
raise OpTestError("Lpar %s not found in managed system %s" %
(self.lpar_name, src_mg_system))
self.set_ssh_key_auth(target_hmc_ip, target_hmc_user,
target_hmc_passwd, remote_hmc)
for mode in ['v', 'm']:
cmd = "migrlpar -o %s -m %s -t %s -p %s -u %s --ip %s %s" % (
mode, src_mg_system, dest_mg_system, self.lpar_name,
target_hmc_user, target_hmc_ip, param)
if options:
cmd = "%s %s" % (cmd, options)
hmc.ssh.run_command(cmd, timeout=timeout)
def recover_lpar(self,
src_mg_system,
dest_mg_system,
stop_lpm=False,
timeout=300):
'''
Recover LPAR
:param src_mg_system: string, Source managed system
:param dest_mg_system: string, Destination managed system
:param stop_lpm: Boolean, to stop migration, default False
:param timeout: number, time in seconds
:returns: True - when LPAR recovery success
False - when LPAR recovery unsuccess
'''
if stop_lpm:
self.ssh.run_command("migrlpar -o s -m %s -p %s" %
(src_mg_system, self.lpar_name),
timeout=timeout)
self.ssh.run_command("migrlpar -o r -m %s -p %s" %
(src_mg_system, self.lpar_name),
timeout=timeout)
if not self.is_lpar_in_managed_system(dest_mg_system, self.lpar_name):
log.info("LPAR recovered at managed system %s" % src_mg_system)
return True
log.info("LPAR failed to recover at managed system %s" % src_mg_system)
return False
def get_adapter_id(self, mg_system, loc_code, remote_hmc=None):
'''
Get SRIOV adapter id
:param mg_system: string, Source managed system
:param loc_code: string, adapater
:param remote_hmc: object, remote HMC instance
:returns: string, adapter id
'''
hmc = remote_hmc if remote_hmc else self
cmd = 'lshwres -m {} -r sriov --rsubtype adapter -F phys_loc:adapter_id'.format(
mg_system)
adapter_id_output = hmc.ssh.run_command(cmd)
for line in adapter_id_output:
if str(loc_code) in line:
return line.split(':')[1]
return ''
def get_lpar_id(self, mg_system, l_lpar_name, remote_hmc=None):
'''
Get LPAR id
:param mg_system: string, Source managed system
:param l_lpar_name: string, LPAR name
:param remote_hmc: object, remote HMC instance
:returns: number, LPAR id on success, 0 on failure
'''
hmc = remote_hmc if remote_hmc else self
cmd = 'lssyscfg -m %s -r lpar --filter lpar_names=%s -F lpar_id' % (
mg_system, l_lpar_name)
lpar_id_output = hmc.ssh.run_command(cmd)
for line in lpar_id_output:
if l_lpar_name in line:
return 0
return line
return 0
def is_msp_enabled(self, mg_system, vios_name, remote_hmc=None):
'''
The function checks if the moving service option is enabled
on the given lpar partition.
:param mg_system: string, Source managed system
:param vios_name: string, VIOS name
:param remote_hmc: object, remote HMC instance
:returns: Boolean, True on success, False on failure
'''
hmc = remote_hmc if remote_hmc else self
cmd = "lssyscfg -m %s -r lpar --filter lpar_names=%s -F msp" % (
mg_system, vios_name)
msp_output = hmc.ssh.run_command(cmd)
if int(msp_output[0]) != 1:
return False
return True
def gather_logs(self,
list_of_commands=[],
remote_hmc=None,
output_dir=None):
'''
Gather the logs for the commands at the given directory
:param list_of_commands: list, list of commands
:param remote_hmc: object, remote HMC instance
:param output_dir: string, dir to store logs
:returns: Boolean, True on success, False on failure
:raises: `CommandFailed`
'''
hmc = remote_hmc if remote_hmc else self
if not output_dir:
output_dir = "HMC_Logs_%s" % (time.asctime(
time.localtime())).replace(" ", "_")
output_dir = os.path.join(self.system.cv_HOST.results_dir, output_dir,
"hmc-" + hmc.hmc_ip)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
default_commands = ['lshmc -V', 'pedbg -c -q 4']
list_of_commands.extend(default_commands)
try:
for cmd in set(list_of_commands):
output = "\n".join(hmc.run_command(r"%s" % cmd, timeout=1800))
filename = "%s.log" % '-'.join(
(re.sub(r'[^a-zA-Z0-9]', ' ', cmd)).split())
filepath = os.path.join(output_dir, filename)
with open(filepath, 'w') as f:
f.write(output)
log.warn(
"Please collect the pedbg logs immediately. At risk of being overwritten."
)
return True
except CommandFailed as cmd_failed:
raise cmd_failed
def run_command_ignore_fail(self, command, timeout=60, retry=0):
'''
Wrapper function for `ssh.run_command_ignore_fail`
:param command: string, command
:param timeout: number, time out in seconds
:param retry: number, number of retries
'''
return self.ssh.run_command_ignore_fail(command,
timeout * self.timeout_factor,
retry)
def run_command(self, i_cmd, timeout=15):
'''
Wrapper function for `ssh.run_command`
:param i_cmd: string, command
:param timeout: number, time out in seconds
'''
return self.ssh.run_command(i_cmd, timeout)