Exemplo n.º 1
0
class ControlC(unittest.TestCase):
    '''
    Start a process that does a bunch of console output, and then try and
    'control-c' it to stop the process and get a prompt back.
    '''
    CONTROL = 'c'

    def setUp(self):
        conf = OpTestConfiguration.conf
        self.cv_BMC = conf.bmc()
        self.cv_SYSTEM = conf.system()
        self.util = OpTestUtil()
        self.prompt = self.util.build_prompt()

    def cleanup(self):
        pass

    def runTest(self):
        self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)
        console = self.cv_BMC.get_host_console()
        if (isinstance(self.cv_BMC, OpTestMambo.OpTestMambo)):
            raise unittest.SkipTest("Mambo so skipping Control-C tests")
        raw_pty = console.get_console()
        raw_pty.sendline("find /")
        time.sleep(2)
        raw_pty.sendcontrol(self.CONTROL)
        BMC_DISCONNECT = 'SOL session closed by BMC'
        timeout = 60
        try:
            rc = raw_pty.expect([BMC_DISCONNECT, self.prompt,
                                     pexpect.TIMEOUT, pexpect.EOF], timeout)
            if rc == 0:
                raise BMCDisconnected(BMC_DISCONNECT)
            self.assertEqual(rc, 1, "Failed to find expected prompt")
        except pexpect.TIMEOUT as e:
            log.debug(e)
            log.debug("# TIMEOUT waiting for command to finish with ctrl-c.")
            log.debug("# Everything is terrible. Fail the world, "
                      "power cycle (if lucky)")
            self.cv_SYSTEM.set_state(OpSystemState.UNKNOWN_BAD)
            self.fail("Could not ctrl-c running command in reasonable time")
        self.cleanup()
Exemplo n.º 2
0
class ControlC(unittest.TestCase):
    CONTROL = 'c'

    def setUp(self):
        conf = OpTestConfiguration.conf
        self.cv_BMC = conf.bmc()
        self.cv_SYSTEM = conf.system()
        self.util = OpTestUtil()
        self.prompt = self.util.build_prompt()

    def cleanup(self):
        pass

    def runTest(self):
        self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)
        console = self.cv_BMC.get_host_console()
        # I should really make this API less nasty...
        raw_console = console.get_console()
        #raw_console.sendline("hexdump -C -v /dev/zero")
        raw_console.sendline("find /")
        time.sleep(2)
        raw_console.sendcontrol(self.CONTROL)
        BMC_DISCONNECT = 'SOL session closed by BMC'
        timeout = 60
        try:
            rc = raw_console.expect(
                [BMC_DISCONNECT, self.prompt, pexpect.TIMEOUT, pexpect.EOF],
                timeout)
            if rc == 0:
                raise BMCDisconnected(BMC_DISCONNECT)
            self.assertEqual(rc, 1, "Failed to find expected prompt")
        except pexpect.TIMEOUT as e:
            log.debug(e)
            log.debug("# TIMEOUT waiting for command to finish with ctrl-c.")
            log.debug(
                "# Everything is terrible. Fail the world, power cycle (if lucky)"
            )
            self.cv_SYSTEM.set_state(OpSystemState.UNKNOWN_BAD)
            self.fail("Could not ctrl-c running command in reasonable time")
        self.cleanup()
Exemplo n.º 3
0
class HMCConsole(HMCUtil):
    """
    HMCConsole Class
    Methods to manage the console of LPAR
    """
    def __init__(self,
                 hmc_ip,
                 user_name,
                 password,
                 managed_system,
                 lpar_name,
                 lpar_vios,
                 lpar_prof,
                 lpar_user,
                 lpar_password,
                 block_setup_term=None,
                 delaybeforesend=None,
                 timeout_factor=1,
                 logfile=sys.stdout,
                 prompt=None,
                 scratch_disk="",
                 check_ssh_keys=False,
                 known_hosts_file=None,
                 proxy=""):
        self.logfile = logfile
        self.hmc_ip = hmc_ip
        self.user = user_name
        self.passwd = password
        self.mg_system = managed_system
        self.util = OpTestUtil()
        self.expect_prompt = self.util.build_prompt(prompt) + "$"
        self.lpar_name = lpar_name
        self.lpar_vios = lpar_vios
        self.lpar_prof = lpar_prof
        self.lpar_user = lpar_user
        self.lpar_password = lpar_password
        self.scratch_disk = scratch_disk
        self.proxy = proxy
        self.state = ConsoleState.DISCONNECTED
        self.delaybeforesend = delaybeforesend
        self.system = None
        # OpTestUtil instance is NOT conf's
        self.prompt = prompt
        self.pty = None
        self.delaybeforesend = delaybeforesend
        # allows caller specific control of when to block setup_term
        self.block_setup_term = block_setup_term
        # tells setup_term to not throw exceptions, like when system off
        self.setup_term_quiet = 0
        # flags the object to abandon setup_term operations, like when system off
        self.setup_term_disable = 0

        # FUTURE - System Console currently tracked in System Object
        # state tracking, reset on boot and state changes
        self.PS1_set = -1
        self.LOGIN_set = -1
        self.SUDO_set = -1
        super(HMCConsole,
              self).__init__(hmc_ip, user_name, password, scratch_disk, proxy,
                             logfile, managed_system, lpar_name, prompt,
                             block_setup_term, delaybeforesend, timeout_factor,
                             lpar_prof, lpar_vios, lpar_user, lpar_password,
                             check_ssh_keys, known_hosts_file)

    def set_system(self, system):
        self.ssh.set_system(system)
        self.system = system
        self.pty = self.get_console()
        self.pty.set_system(system)

    def get_host_console(self):
        return self.pty

    def set_system_setup_term(self, flag):
        self.system.block_setup_term = flag

    def get_system_setup_term(self):
        return self.system.block_setup_term

    def get_scratch_disk(self):
        return self.scratch_disk

    def get_proxy(self):
        return self.proxy

    def hostname(self):
        return self.hmc_ip

    def username(self):
        return self.user

    def password(self):
        return self.passwd

    def set_block_setup_term(self, flag):
        self.block_setup_term = flag

    def get_block_setup_term(self):
        return self.block_setup_term

    def enable_setup_term_quiet(self):
        self.setup_term_quiet = 1
        self.setup_term_disable = 0

    def disable_setup_term_quiet(self):
        self.setup_term_quiet = 0
        self.setup_term_disable = 0

    def close(self):
        self.util.clear_state(self)
        try:
            self.pty.close()
            if self.pty.status != -1:  # leaving for debug
                if os.WIFEXITED(self.pty.status):
                    os.WEXITSTATUS(self.pty.status)
                else:
                    os.WTERMSIG(self.pty.status)
            self.state = ConsoleState.DISCONNECTED
        except pexpect.ExceptionPexpect:
            self.state = ConsoleState.DISCONNECTED
            raise "HMC Console: failed to close console"
        except Exception:
            self.state = ConsoleState.DISCONNECTED
        log.debug("HMC close -> TERMINATE")

    def connect(self, logger=None):
        if self.state == ConsoleState.CONNECTED:
            return self.pty
        self.util.clear_state(self)  # clear when coming in DISCONNECTED

        log.info("De-activating the console")
        self.deactivate_lpar_console()

        log.debug("#HMC Console CONNECT")

        command = "sshpass -p %s ssh -p 22 -l %s %s -o PubkeyAuthentication=no"\
                  " -o afstokenpassing=no -q -o 'UserKnownHostsFile=/dev/null'"\
                  " -o 'StrictHostKeyChecking=no'"
        try:
            self.pty = Spawn(command % (self.passwd, self.user, self.hmc_ip))
            log.info("Opening the LPAR console")
            time.sleep(STALLTIME)
            self.pty.send('\r')
            self.pty.sendline("mkvterm -m %s -p %s" %
                              (self.mg_system, self.lpar_name))
            self.pty.send('\r')
            time.sleep(STALLTIME)
            i = self.pty.expect(["Open Completed.", pexpect.TIMEOUT],
                                timeout=60)
            self.pty.logfile = sys.stdout
            if logger:
                self.pty.logfile_read = OpTestLogger.FileLikeLogger(logger)
            else:
                self.pty.logfile_read = OpTestLogger.FileLikeLogger(log)

            if i == 0:
                time.sleep(STALLTIME)
                self.state = ConsoleState.CONNECTED
                self.pty.setwinsize(1000, 1000)
            else:
                raise OpTestError("Check the lpar activate command")
        except Exception as exp:
            self.state = ConsoleState.DISCONNECTED
            raise CommandFailed(
                'OPexpect.spawn',
                'OPexpect.spawn encountered a problem: ' + str(exp), -1)

        if self.delaybeforesend:
            self.pty.delaybeforesend = self.delaybeforesend

        if not self.pty.isalive():
            raise CommandFailed("mkvterm", self.pty.read(), self.pty.status)
        return self.pty

    def check_state(self):
        return self.state

    def get_console(self, logger=None):
        if self.state == ConsoleState.DISCONNECTED:
            self.util.clear_state(self)
            self.connect(logger=logger)
            time.sleep(STALLTIME)
            l_rc = self.pty.expect(["login:"******"Shell prompt changed")
                else:
                    self.pty.send('\r')
                    log.debug("Waiting till booting!")
                    self.pty = self.get_login_prompt()

        if self.system.SUDO_set != 1 or self.system.LOGIN_set != 1 or self.system.PS1_set != 1:
            self.util.setup_term(self.system, self.pty, None,
                                 self.system.block_setup_term)
        # Clear buffer before usage
        self.pty.buffer = ""
        return self.pty

    def get_login_prompt(self):
        # Assuming 'Normal' boot set in LPAR profile
        # We wait for upto 500 seconds for LPAR to boot to OS
        self.pty.send('\r')
        time.sleep(STALLTIME)
        log.debug("Waiting for login screen")
        i = self.pty.expect(["login:"******"System has booted")
            time.sleep(STALLTIME)
        elif i == 1:
            log.debug("Console already logged in")
        else:
            log.error("Failed to get login prompt %s", self.pty.before)
            # To cheat system for making using of HMC SSH
            self.system.PS1_set = 1
            self.system.LOGIN_set = 1
            self.system.SUDO_set = 1
        return self.pty

    def run_command(self, i_cmd, timeout=15):
        return self.util.run_command(self, i_cmd, timeout)
Exemplo n.º 4
0
class HMCUtil():
    '''
    Utility and functions of HMC object
    '''
    def __init__(self,
                 hmc_ip,
                 user_name,
                 password,
                 scratch_disk="",
                 proxy="",
                 logfile=sys.stdout,
                 managed_system=None,
                 lpar_name=None,
                 prompt=None,
                 block_setup_term=None,
                 delaybeforesend=None,
                 timeout_factor=None,
                 lpar_prof=None,
                 lpar_vios=None,
                 lpar_user=None,
                 lpar_password=None,
                 check_ssh_keys=False,
                 known_hosts_file=None,
                 tgt_managed_system=None):
        self.hmc_ip = hmc_ip
        self.user = user_name
        self.passwd = password
        self.logfile = logfile
        self.mg_system = managed_system
        self.tgt_mg_system = tgt_managed_system
        self.check_ssh_keys = check_ssh_keys
        self.known_hosts_file = known_hosts_file
        self.lpar_name = lpar_name
        self.lpar_prof = lpar_prof
        self.lpar_user = lpar_user
        self.lpar_password = lpar_password
        self.lpar_vios = lpar_vios
        self.util = OpTestUtil()
        self.prompt = prompt
        self.expect_prompt = self.util.build_prompt(prompt) + "$"
        self.ssh = OpTestSSH(hmc_ip,
                             user_name,
                             password,
                             logfile=self.logfile,
                             check_ssh_keys=check_ssh_keys,
                             known_hosts_file=known_hosts_file,
                             block_setup_term=block_setup_term)
        self.scratch_disk = scratch_disk
        self.proxy = proxy
        self.scratch_disk_size = None
        self.delaybeforesend = delaybeforesend
        self.system = None
        # OpTestUtil instance is NOT conf's
        self.pty = None
        # allows caller specific control of when to block setup_term
        self.block_setup_term = block_setup_term
        # tells setup_term to not throw exceptions, like when system off
        self.setup_term_quiet = 0
        # flags the object to abandon setup_term operations, like when system off
        self.setup_term_disable = 0
        # functional simulators are very slow, so multiply all default timeouts by this factor
        self.timeout_factor = timeout_factor

        # state tracking, reset on boot and state changes
        # console tracking done on System object for the system console
        self.PS1_set = -1
        self.LOGIN_set = -1
        self.SUDO_set = -1

    def deactivate_lpar_console(self):
        self.ssh.run_command("rmvterm -m %s -p %s" %
                             (self.mg_system, self.lpar_name),
                             timeout=10)

    def poweroff_system(self):
        if self.get_system_state() != OpManagedState.OPERATING:
            raise OpTestError('Managed Systen not in Operating state')
        self.ssh.run_command("chsysstate -m %s -r sys -o off" % self.mg_system)
        self.wait_system_state(OpManagedState.OFF)

    def poweron_system(self):
        if self.get_system_state() != OpManagedState.OFF:
            raise OpTestError('Managed Systen not is Power off state!')
        self.ssh.run_command("chsysstate -m %s -r sys -o on" % self.mg_system)
        self.wait_system_state()
        if self.lpar_vios:
            log.debug("Starting VIOS %s", self.lpar_vios)
            self.poweron_lpar(vios=True)

    def poweroff_lpar(self):
        if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
            log.info('LPAR Already powered-off!')
            return
        self.ssh.run_command(
            "chsysstate -m %s -r lpar -n %s -o shutdown --immed" %
            (self.mg_system, self.lpar_name))
        self.wait_lpar_state(OpHmcState.NOT_ACTIVE)

    def poweron_lpar(self, vios=False):
        if self.get_lpar_state(vios) == OpHmcState.RUNNING:
            log.info('LPAR Already powered on!')
            return BMC_CONST.FW_SUCCESS
        lpar_name = self.lpar_name
        if vios:
            lpar_name = self.lpar_vios
        cmd = "chsysstate -m %s -r lpar -n %s -o on" % (self.mg_system,
                                                        lpar_name)

        if self.lpar_prof:
            cmd = "%s -f %s" % (cmd, self.lpar_prof)

        self.wait_lpar_state(OpHmcState.NOT_ACTIVE, vios=vios)
        self.ssh.run_command(cmd)
        self.wait_lpar_state(vios=vios)
        time.sleep(STALLTIME)
        return BMC_CONST.FW_SUCCESS

    def dumprestart_lpar(self):
        if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
            log.info('LPAR Already powered-off!')
            return
        self.ssh.run_command("chsysstate -m %s -r lpar -n %s -o dumprestart" %
                             (self.mg_system, self.lpar_name))
        self.wait_lpar_state()

    def restart_lpar(self):
        if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
            log.info('LPAR Already powered-off!')
            return
        self.ssh.run_command(
            "chsysstate -m %s -r lpar -n %s -o shutdown --immed --restart" %
            (self.mg_system, self.lpar_name))
        self.wait_lpar_state()

    def get_lpar_state(self, vios=False):
        lpar_name = self.lpar_name
        if vios:
            lpar_name = self.lpar_vios
        state = self.ssh.run_command(
            'lssyscfg -m %s -r lpar --filter lpar_names=%s -F state' %
            (self.mg_system, lpar_name))[-1]
        ref_code = self.ssh.run_command(
            'lsrefcode -m %s -r lpar --filter lpar_names=%s -F refcode' %
            (self.mg_system, lpar_name))[-1]
        if state == 'Running':
            if 'Linux' in ref_code or not ref_code:
                return 'Running'
            else:
                return 'Booting'
        return state

    def get_system_state(self):
        state = self.ssh.run_command('lssyscfg -m %s -r sys -F state' %
                                     self.mg_system)
        return state[-1]

    def wait_lpar_state(self,
                        exp_state=OpHmcState.RUNNING,
                        vios=False,
                        timeout=WAITTIME):
        state = self.get_lpar_state(vios)
        count = 0
        while state != exp_state:
            state = self.get_lpar_state(vios)
            log.info("Current state: %s", state)
            time.sleep(timeout)
            count = 1
            if count > 120:
                raise OpTestError("Time exceeded for reaching %s" % exp_state)

    def wait_system_state(self,
                          exp_state=OpManagedState.OPERATING,
                          timeout=WAITTIME):
        state = self.get_system_state()
        count = 0
        while state != exp_state:
            state = self.get_system_state()
            log.info("Current state: %s", state)
            time.sleep(timeout)
            count = 1
            if count > 60:
                raise OpTestError("Time exceeded for reaching %s" % exp_state)

    def is_lpar_in_managed_system(self, mg_system=None):
        lpar_list = self.ssh.run_command('lssyscfg -r lpar -m %s -F name' %
                                         mg_system)
        if self.lpar_name in lpar_list:
            log.info("%s lpar found in managed system %s" %
                     (mg_system, self.lpar_name))
            return True
        return False

    def migrate_lpar(self, src_mg_system=None, dest_mg_system=None):
        if src_mg_system == None or dest_mg_system == None:
            raise OpTestError(
                "Source and Destination Managed System required for LPM")
        if not self.is_lpar_in_managed_system(src_mg_system):
            raise OpTestError("Lpar %s not found in managed system %s" %
                              (self.lpar_name, src_mg_system))
        self.ssh.run_command('migrlpar -o v -m %s -t %s -p %s' %
                             (src_mg_system, dest_mg_system, self.lpar_name))
        self.ssh.run_command('migrlpar -o m -m %s -t %s -p %s' %
                             (src_mg_system, dest_mg_system, self.lpar_name))
        if self.is_lpar_in_managed_system(dest_mg_system):
            log.info("Migration of lpar %s from %s to %s is successfull" %
                     (self.lpar_name, src_mg_system, dest_mg_system))
            self.mg_system = dest_mg_system
            return True
        log.info("Migration of lpar %s from %s to %s failed" %
                 (self.lpar_name, src_mg_system, dest_mg_system))
        return False

    def run_command_ignore_fail(self, command, timeout=60, retry=0):
        return self.ssh.run_command_ignore_fail(command,
                                                timeout * self.timeout_factor,
                                                retry)

    def run_command(self, i_cmd, timeout=15):
        return self.ssh.run_command(i_cmd, timeout)
Exemplo n.º 5
0
class OpTestHMC():
    '''
    This class contains the modules to perform various HMC operations on an LPAR.
    The Host IP, username and password of HMC have to be passed to the class intially
    while creating the object for the class.
    '''
    def __init__(self,
                 hmc_ip,
                 user_name,
                 password,
                 scratch_disk="",
                 proxy="",
                 logfile=sys.stdout,
                 managed_system=None,
                 lpar_name=None,
                 prompt=None,
                 lpar_prof=None,
                 lpar_vios=None,
                 lpar_user=None,
                 lpar_password=None,
                 check_ssh_keys=False,
                 known_hosts_file=None):
        self.hmc_ip = hmc_ip
        self.user = user_name
        self.passwd = password
        self.logfile = logfile
        self.system = managed_system
        self.system = managed_system
        self.check_ssh_keys = check_ssh_keys
        self.known_hosts_file = known_hosts_file
        self.lpar_name = lpar_name
        self.lpar_prof = lpar_prof
        self.lpar_user = lpar_user
        self.lpar_password = lpar_password
        self.lpar_vios = lpar_vios
        self.lpar_con = None
        self.vterm = False
        self.util = OpTestUtil()
        self.prompt = prompt
        self.expect_prompt = self.util.build_prompt(prompt) + "$"
        self.conf = OpTestConfiguration.conf
        self.ssh = OpTestSSH(hmc_ip,
                             user_name,
                             password,
                             logfile=self.logfile,
                             check_ssh_keys=check_ssh_keys,
                             known_hosts_file=known_hosts_file)
        self.scratch_disk = scratch_disk
        self.proxy = proxy
        self.scratch_disk_size = None

    def hostname(self):
        return self.hmc_ip

    def username(self):
        return self.user

    def password(self):
        return self.passwd

    def set_system(self, system):
        self.ssh.set_system(system)

    def get_scratch_disk(self):
        return self.scratch_disk

    def run_command(self, i_cmd, timeout=15):
        return self.ssh.run_command(i_cmd, timeout)

    def deactivate_lpar_console(self):
        self.run_command("rmvterm -m %s -p %s" % (self.system, self.lpar_name),
                         timeout=10)

    def poweroff_system(self):
        if self.get_system_state() != OpManagedState.OPERATING:
            raise OpTestError('Managed Systen not in Operating state')
        self.run_command("chsysstate -m %s -r sys -o off" % self.system)
        self.wait_system_state(OpManagedState.OFF)

    def poweron_system(self):
        if self.get_system_state() != OpManagedState.OFF:
            raise OpTestError('Managed Systen not is Power off state!')
        self.run_command("chsysstate -m %s -r sys -o on" % self.system)
        self.wait_system_state()
        if self.lpar_vios:
            log.debug("Starting VIOS %s" % self.lpar_vios)
            self.poweron_lpar(vios=True)

    def poweroff_lpar(self):
        if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
            log.info('LPAR Already powered-off!')
            return
        self.run_command("chsysstate -m %s -r lpar -n %s -o shutdown --immed" %
                         (self.system, self.lpar_name))
        self.wait_lpar_state(OpHmcState.NOT_ACTIVE)

    def poweron_lpar(self, runtime=False, vios=False):
        if self.get_lpar_state(vios) == OpHmcState.RUNNING:
            log.info('LPAR Already powered on!')
            return
        lpar_name = self.lpar_name
        if vios:
            lpar_name = self.lpar_vios
        cmd = "chsysstate -m %s -r lpar -n %s -o on" % (self.system, lpar_name)

        if not vios:
            if self.lpar_prof:
                cmd = "%s -f %s" % (cmd, self.lpar_prof)

        self.wait_lpar_state(OpHmcState.NOT_ACTIVE, vios=vios)
        self.run_command(cmd)
        self.wait_lpar_state(vios=vios)
        if runtime:
            self.wait_login_prompt(self.get_console_prompt())
            self.close_console(self.lpar_con)

    def get_lpar_state(self, vios=False):
        lpar_name = self.lpar_name
        if vios:
            lpar_name = self.lpar_vios
        state = self.run_command(
            'lssyscfg -m %s -r lpar --filter lpar_names=%s -F state' %
            (self.system, lpar_name))
        return state[-1]

    def get_system_state(self):
        state = self.run_command('lssyscfg -m %s -r sys -F state' %
                                 self.system)
        return state[-1]

    def wait_lpar_state(self,
                        exp_state=OpHmcState.RUNNING,
                        vios=False,
                        timeout=WAITTIME):
        state = self.get_lpar_state(vios)
        count = 0
        while state != exp_state:
            state = self.get_lpar_state(vios)
            log.info("Current state: %s" % state)
            time.sleep(timeout)
            count = 1
            if count > 120:
                raise OpTestError("Time exceeded for reaching %s" % exp_state)

    def wait_system_state(self,
                          exp_state=OpManagedState.OPERATING,
                          timeout=WAITTIME):
        state = self.get_system_state()
        count = 0
        while state != exp_state:
            state = self.get_system_state()
            log.info("Current state: %s" % state)
            time.sleep(timeout)
            count = 1
            if count > 60:
                raise OpTestError("Time exceeded for reaching %s" % exp_state)

    def vterm_run_command(self, console, cmd, timeout=60):
        if console is None:
            console = self.get_console()

        try:
            console.send(cmd)
            console.send('\r')
            output = console.before
            console.sendline("echo $?")
            console.send('\r')
            # Time to accumulate both outputs
            time.sleep(STALLTIME)
            rc = console.expect([self.expect_prompt], timeout=timeout)
            if rc != 0:
                exitcode = int(console.before)
                log.debug("# LAST COMMAND EXIT CODE %d (%s)" %
                          (exitcode, repr(console.before)))
        except pexpect.TIMEOUT as e:
            log.debug(e)
            log.debug("# TIMEOUT waiting for command to finish.")
            log.debug("# Attempting to control-c")
            try:
                console.sendcontrol('c')
                rc = console.expect([self.expect_prompt], 10)
                if rc == 0:
                    raise CommandFailed(cmd, "TIMEOUT", -1)
            except pexpect.TIMEOUT:
                log.error("# Timeout trying to kill timed-out command.")
                log.error(
                    "# Failing current command and attempting to continue")
                self.deactivate_lpar_console()
                raise CommandFailed("console", "timeout", -1)
            raise e

        if rc == 0:
            res = console.before
            res = res.split(cmd)
            return res[-1].splitlines()[1:-2]
        else:
            res = output
            res = res.splitlines()
            if exitcode != 0:
                raise CommandFailed(cmd, res, exitcode)
            return res

    def get_console(self):
        console = self.get_console_prompt()
        time.sleep(STALLTIME)
        l_rc = console.expect(["login:"******"Shell prompt changed")
            else:
                console.send('\r')
                log.debug("Waiting till booting!")
                console = self.wait_login_prompt(console, self.lpar_user,
                                                 self.lpar_password)
        return console

    def check_vterm(self):
        return self.vterm

    def wait_login_prompt(self, console, username=None, password=None):
        if not console:
            raise OpTestError("Console is not provided")
        # Assuming 'Normal' boot set in LPAR profile
        # We wait for upto 500 seconds for LPAR to boot to OS
        console.buffer = ""
        console.send('\r')
        time.sleep(STALLTIME)
        log.debug("Waiting for login screen")
        i = console.expect(["login:"******"System has booted")
            if username and password:
                self.lpar_con.sendline(username)
                self.lpar_con.send('\r')
                time.sleep(STALLTIME)
                i = self.lpar_con.expect(["Password:"******"Last login", "incorrect", pexpect.TIMEOUT],
                        timeout=60)
                    if i == 0:
                        log.info('Logged in to console')
                        time.sleep(STALLTIME)
                        return self.lpar_con
                    elif i == 1:
                        raise OpTestError("Wrong Credentials for host")
                    else:
                        raise OpTestError("Upexpected return")
            time.sleep(STALLTIME)
            return self.lpar_con
        elif i == 1:
            # Assuming console already logged in !"
            return self.lpar_con
        else:
            log.error("%s %s" % (i, self.lpar_con.before))
            raise OpTestError("Console in different state")

    def get_console_prompt(self):
        if self.get_lpar_state() != OpHmcState.RUNNING:
            raise OpTestError('LPAR is not in Running State. Please check!')
        log.info("De-activating the console")

        self.deactivate_lpar_console()
        command = "sshpass -p %s ssh -p 22 -l %s %s -o PubkeyAuthentication=no"\
                  " -o afstokenpassing=no -q -o 'UserKnownHostsFile=/dev/null'"\
                  " -o 'StrictHostKeyChecking=no'"
        self.lpar_con = OPexpect.spawn(command %
                                       (self.passwd, self.user, self.hmc_ip))
        self.wait_lpar_state()
        log.info("Opening the LPAR console")
        self.vterm = True
        self.lpar_con.sendline("mkvterm -m %s -p %s" %
                               (self.system, self.lpar_name))
        self.lpar_con.send('\r')
        time.sleep(STALLTIME)
        i = self.lpar_con.expect(["Open Completed.", pexpect.TIMEOUT],
                                 timeout=60)
        self.lpar_con.logfile = sys.stdout
        self.lpar_con.logfile_read = OpTestLogger.FileLikeLogger(log)
        if i == 0:
            time.sleep(STALLTIME)
            return self.lpar_con
        else:
            raise OpTestError("Check the lpar activate command")

    def close_console(self, console):
        console.send('~.')
        self.vterm = False
        time.sleep(STALLTIME)
        console.close()
Exemplo n.º 6
0
class HMCUtil():
    '''
    Utility and functions of HMC object
    '''
    def __init__(self,
                 hmc_ip,
                 user_name,
                 password,
                 scratch_disk="",
                 proxy="",
                 logfile=sys.stdout,
                 managed_system=None,
                 lpar_name=None,
                 prompt=None,
                 block_setup_term=None,
                 delaybeforesend=None,
                 timeout_factor=None,
                 lpar_prof=None,
                 lpar_vios=None,
                 lpar_user=None,
                 lpar_password=None,
                 check_ssh_keys=False,
                 known_hosts_file=None,
                 tgt_managed_system=None,
                 tgt_lpar=None):
        self.hmc_ip = hmc_ip
        self.user = user_name
        self.passwd = password
        self.logfile = logfile
        self.mg_system = managed_system
        self.tgt_mg_system = tgt_managed_system
        self.tgt_lpar = tgt_lpar
        self.check_ssh_keys = check_ssh_keys
        self.known_hosts_file = known_hosts_file
        self.lpar_name = lpar_name
        self.lpar_prof = lpar_prof
        self.lpar_user = lpar_user
        self.lpar_password = lpar_password
        self.lpar_vios = lpar_vios
        self.util = OpTestUtil()
        self.prompt = prompt
        self.expect_prompt = self.util.build_prompt(prompt) + "$"
        self.ssh = OpTestSSH(hmc_ip,
                             user_name,
                             password,
                             logfile=self.logfile,
                             check_ssh_keys=check_ssh_keys,
                             known_hosts_file=known_hosts_file,
                             block_setup_term=block_setup_term)
        self.scratch_disk = scratch_disk
        self.proxy = proxy
        self.scratch_disk_size = None
        self.delaybeforesend = delaybeforesend
        self.system = None
        # OpTestUtil instance is NOT conf's
        self.pty = None
        # allows caller specific control of when to block setup_term
        self.block_setup_term = block_setup_term
        # tells setup_term to not throw exceptions, like when system off
        self.setup_term_quiet = 0
        # flags the object to abandon setup_term operations, like when system off
        self.setup_term_disable = 0
        # functional simulators are very slow, so multiply all default timeouts by this factor
        self.timeout_factor = timeout_factor

        # state tracking, reset on boot and state changes
        # console tracking done on System object for the system console
        self.PS1_set = -1
        self.LOGIN_set = -1
        self.SUDO_set = -1

    def deactivate_lpar_console(self):
        '''
        Deactivate/disconnect the LPAR Console
        '''
        self.ssh.run_command("rmvterm -m %s -p %s" %
                             (self.mg_system, self.lpar_name),
                             timeout=10)

    def poweroff_system(self):
        '''
        PowerOFF the managed system
        '''
        if self.get_system_state() != OpManagedState.OPERATING:
            raise OpTestError('Managed Systen not in Operating state')
        self.ssh.run_command("chsysstate -m %s -r sys -o off" % self.mg_system)
        self.wait_system_state(OpManagedState.OFF)

    def poweron_system(self):
        '''
        PowerON the managed system
        '''
        if self.get_system_state() != OpManagedState.OFF:
            raise OpTestError('Managed Systen not is Power off state!')
        self.ssh.run_command("chsysstate -m %s -r sys -o on" % self.mg_system)
        self.wait_system_state()
        if self.lpar_vios:
            log.debug("Starting VIOS %s", self.lpar_vios)
            self.poweron_lpar(vios=True)

    def poweroff_lpar(self, remote_hmc=None):
        '''
        PowerOFF the LPAR

        :param remote_hmc: object, remote HMC instance
        '''
        hmc = remote_hmc if remote_hmc else self
        if self.get_lpar_state(remote_hmc=remote_hmc) in [
                OpHmcState.NOT_ACTIVE, OpHmcState.NA
        ]:
            log.info('LPAR Already powered-off!')
            return
        hmc.ssh.run_command(
            "chsysstate -m %s -r lpar -n %s -o shutdown --immed" %
            (hmc.mg_system, self.lpar_name))
        self.wait_lpar_state(OpHmcState.NOT_ACTIVE, remote_hmc=remote_hmc)

    def poweron_lpar(self, vios=False, remote_hmc=None):
        '''
        PowerON the LPAR

        :param vios: Boolean, to identify VIOS partition
        :param remote_hmc: object, remote HMC instance
        :returns: BMC_CONST.FW_SUCCESS up on success
        '''
        hmc = remote_hmc if remote_hmc else self
        if self.get_lpar_state(vios,
                               remote_hmc=remote_hmc) == OpHmcState.RUNNING:
            log.info('LPAR Already powered on!')
            return BMC_CONST.FW_SUCCESS
        lpar_name = self.lpar_name
        if vios:
            lpar_name = hmc.lpar_vios
        cmd = "chsysstate -m %s -r lpar -n %s -o on" % (hmc.mg_system,
                                                        lpar_name)

        if self.lpar_prof:
            cmd = "%s -f %s" % (cmd, self.lpar_prof)

        self.wait_lpar_state(OpHmcState.NOT_ACTIVE,
                             vios=vios,
                             remote_hmc=remote_hmc)
        hmc.ssh.run_command(cmd)
        self.wait_lpar_state(vios=vios, remote_hmc=remote_hmc)
        time.sleep(STALLTIME)
        return BMC_CONST.FW_SUCCESS

    def dumprestart_lpar(self):
        '''
        Capture dump and restart the LPAR
        '''
        if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
            log.info('LPAR Already powered-off!')
            return
        self.ssh.run_command("chsysstate -m %s -r lpar -n %s -o dumprestart" %
                             (self.mg_system, self.lpar_name))
        self.wait_lpar_state()

    def restart_lpar(self):
        '''
        Restart LPAR immediately
        '''
        if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
            log.info('LPAR Already powered-off!')
            return
        self.ssh.run_command(
            "chsysstate -m %s -r lpar -n %s -o shutdown --immed --restart" %
            (self.mg_system, self.lpar_name))
        self.wait_lpar_state()

    def get_lpar_cfg(self):
        '''
        Get LPAR configuration parameters

        :returns: LPAR configuration parameters in key, value pair
        '''
        out = self.ssh.run_command(
            "lssyscfg -r prof -m %s --filter 'lpar_names=%s'" %
            (self.mg_system, self.lpar_name))[-1]
        cfg_dict = {}
        splitter = shlex.shlex(out)
        splitter.whitespace += ','
        splitter.whitespace_split = True
        for values in list(splitter):
            data = values.split("=")
            key = data[0]
            value = data[1]
            cfg_dict[key] = value
        return cfg_dict

    def set_lpar_cfg(self, arg_str):
        '''
        Set LPAR configuration parameter values

        :param arg_str: configuration values in key, value pair seperated by
                        comma
        '''
        if not self.lpar_prof:
            raise OpTestError("Profile needs to be defined to use this method")
        self.ssh.run_command(
            "chsyscfg -r prof -m %s -p %s -i 'lpar_name=%s,name=%s,%s' --force"
            % (self.mg_system, self.lpar_name, self.lpar_name, self.lpar_prof,
               arg_str))

    def get_lpar_state(self, vios=False, remote_hmc=None):
        '''
        Get current state of LPAR

        :param vios: Boolean, to identify VIOS partition
        :param remote_hmc: object, remote HMC instance
        :returns: the current status of the LPAR e.g 'Running' or 'Booting'
        '''
        hmc = remote_hmc if remote_hmc else self
        lpar_name = self.lpar_name
        if vios:
            lpar_name = hmc.lpar_vios
        state = hmc.ssh.run_command(
            'lssyscfg -m %s -r lpar --filter lpar_names=%s -F state' %
            (hmc.mg_system, lpar_name))[-1]
        ref_code = hmc.ssh.run_command(
            'lsrefcode -m %s -r lpar --filter lpar_names=%s -F refcode' %
            (hmc.mg_system, lpar_name))[-1]
        if state == 'Running':
            if 'Linux' in ref_code or not ref_code:
                return 'Running'
            else:
                return 'Booting'
        return state

    def get_system_state(self):
        '''
        Get current state of managed system

        :returns: the current status of the managed system
        '''
        state = self.ssh.run_command('lssyscfg -m %s -r sys -F state' %
                                     self.mg_system)
        return state[-1]

    def wait_lpar_state(self,
                        exp_state=OpHmcState.RUNNING,
                        vios=False,
                        timeout=WAITTIME,
                        remote_hmc=None):
        '''
        Wait for a particular state of LPAR

        :param exp_state: constant, expected HMC STATE, default is OpHmcState.RUNNING
        :param vios: Boolean, to identify VIOS partition
        :param timeout: number, Wait time in seconds
        :param remote_hmc: object, remote HMC instance
        :raises: :class:`common.OpTestError` when the timeout happens
        '''
        state = self.get_lpar_state(vios, remote_hmc=remote_hmc)
        count = 0
        while state != exp_state:
            state = self.get_lpar_state(vios, remote_hmc=remote_hmc)
            log.info("Current state: %s", state)
            time.sleep(timeout)
            count += 1
            if count > 120:
                raise OpTestError("Time exceeded for reaching %s" % exp_state)

    def wait_system_state(self,
                          exp_state=OpManagedState.OPERATING,
                          timeout=SYS_WAITTIME):
        '''
        Wait for a particular state of managed system

        :param exp_state: constatn, expected STATE, default is OpManagedState.OPERATING
        :param timeout: number, Wait time in seconds
        :raises: :class:`common.OpTestError` when the timeout happens
        '''
        state = self.get_system_state()
        count = 0
        while state != exp_state:
            state = self.get_system_state()
            log.info("Current state: %s", state)
            time.sleep(timeout)
            count += 1
            if count > 60:
                raise OpTestError("Time exceeded for reaching %s" % exp_state)

    def is_lpar_in_managed_system(self,
                                  mg_system=None,
                                  lpar_name=None,
                                  remote_hmc=None):
        '''
        Check if LPAR is available on a managed system

        :param mg_system: string, managed system name
        :param lpar_name: string, LPAR name
        :param remote_hmc: object, remote HMC instance
        :returns: True  - when the LPAR is in managed system
                  False - when the LPAR is not in managed system
        '''
        hmc = remote_hmc if remote_hmc else self
        lpar_list = hmc.ssh.run_command('lssyscfg -r lpar -m %s -F name' %
                                        mg_system)
        if lpar_name in lpar_list:
            log.info("%s lpar found in managed system %s" %
                     (lpar_name, mg_system))
            return True
        return False

    def migrate_lpar(self,
                     src_mg_system=None,
                     dest_mg_system=None,
                     options=None,
                     param="",
                     timeout=300):
        '''
        Migrate the LPAR from source managed system to the destination managed
        system

        :param src_mg_system: string, source managed system name
        :param dest_mg_system: string, destination manages system name
        :param options: string, options for migration
        :param param: string, required parameters for migration
        :param timeout: number, time out value in seconds, default 300
        :returns: True  - when LPAR migration success
                  False - when LPAR migration unsuccess
        '''
        if src_mg_system == None or dest_mg_system == None:
            raise OpTestError(
                "Source and Destination Managed System required for LPM")
        if not self.is_lpar_in_managed_system(src_mg_system, self.lpar_name):
            raise OpTestError("Lpar %s not found in managed system %s" %
                              (self.lpar_name, src_mg_system))
        for mode in ['v', 'm']:
            cmd = 'migrlpar -o %s -m %s -t %s -p %s %s' % (
                mode, src_mg_system, dest_mg_system, self.lpar_name, param)
            if options:
                cmd = "%s %s" % (cmd, options)
            self.ssh.run_command(cmd, timeout=timeout)
        log.debug("Waiting for %.2f minutes." % (timeout / 60))
        time.sleep(timeout)
        if self.is_lpar_in_managed_system(dest_mg_system, self.lpar_name):
            cmd = "lssyscfg -m %s -r lpar --filter lpar_names=%s -F state" % (
                dest_mg_system, self.lpar_name)
            lpar_state = self.ssh.run_command(cmd)[0]
            if lpar_state not in [
                    'Migrating - Running', 'Migrating - Not Activated'
            ]:
                log.info("Migration of lpar %s from %s to %s is successfull" %
                         (self.lpar_name, src_mg_system, dest_mg_system))
                self.mg_system = dest_mg_system
                return True
            self.recover_lpar(src_mg_system,
                              dest_mg_system,
                              stop_lpm=True,
                              timeout=timeout)
        log.info("Migration of lpar %s from %s to %s failed" %
                 (self.lpar_name, src_mg_system, dest_mg_system))
        return False

    def set_ssh_key_auth(self, hmc_ip, hmc_user, hmc_passwd, remote_hmc=None):
        '''
        Set SSH authentication keys

        :param hmc_ip: number, in ip address format
        :param hmc_user: string, HMC user name
        :param hmc_passwd: string, HMC user password
        :param remote_hmc: object, remote HMC instance
        :raises: `CommandFailed`
        '''
        hmc = remote_hmc if remote_hmc else self
        try:
            cmd = "mkauthkeys -u %s --ip %s --test" % (hmc_user, hmc_ip)
            hmc.run_command(cmd, timeout=120)
        except CommandFailed:
            try:
                cmd = "mkauthkeys -u %s --ip %s --passwd %s" % (
                    hmc_user, hmc_ip, hmc_passwd)
                hmc.run_command(cmd, timeout=120)
            except CommandFailed as cf:
                raise cf

    def cross_hmc_migration(self,
                            src_mg_system=None,
                            dest_mg_system=None,
                            target_hmc_ip=None,
                            target_hmc_user=None,
                            target_hmc_passwd=None,
                            remote_hmc=None,
                            options=None,
                            param="",
                            timeout=300):
        '''
        migration of LPAR across HMC

        :param src_mg_system: string, Source managed system
        :param dest_mg_system: string, Destination managed system
        :param target_hmc_ip: number, in IP address format
        :param target_hmc_user: string, target HMC user name
        :param target_hmc_passwd: string, target HMC user password
        :param remote_hmc: object, remote HMC instance
        :param options: string, migration options
        :param param: string, migration parameters
        :param timeout: number, time in seconds
        '''
        hmc = remote_hmc if remote_hmc else self

        if src_mg_system == None or dest_mg_system == None:
            raise OpTestError("Source and Destination Managed System "\
            "required for Cross HMC LPM")
        if target_hmc_ip == None or target_hmc_user == None or target_hmc_passwd == None:
            raise OpTestError("Destination HMC IP, Username, and Password "\
            "required for Cross HMC LPM")
        if not self.is_lpar_in_managed_system(src_mg_system, self.lpar_name,
                                              remote_hmc):
            raise OpTestError("Lpar %s not found in managed system %s" %
                              (self.lpar_name, src_mg_system))

        self.set_ssh_key_auth(target_hmc_ip, target_hmc_user,
                              target_hmc_passwd, remote_hmc)

        for mode in ['v', 'm']:
            cmd = "migrlpar -o %s -m %s -t %s -p %s -u %s --ip %s %s" % (
                mode, src_mg_system, dest_mg_system, self.lpar_name,
                target_hmc_user, target_hmc_ip, param)
            if options:
                cmd = "%s %s" % (cmd, options)
            hmc.ssh.run_command(cmd, timeout=timeout)

    def recover_lpar(self,
                     src_mg_system,
                     dest_mg_system,
                     stop_lpm=False,
                     timeout=300):
        '''
        Recover LPAR

        :param src_mg_system: string, Source managed system
        :param dest_mg_system: string, Destination managed system
        :param stop_lpm: Boolean, to stop migration, default False
        :param timeout: number, time in seconds
        :returns: True  - when LPAR recovery success
                  False - when LPAR recovery unsuccess
        '''
        if stop_lpm:
            self.ssh.run_command("migrlpar -o s -m %s -p %s" %
                                 (src_mg_system, self.lpar_name),
                                 timeout=timeout)
        self.ssh.run_command("migrlpar -o r -m %s -p %s" %
                             (src_mg_system, self.lpar_name),
                             timeout=timeout)
        if not self.is_lpar_in_managed_system(dest_mg_system, self.lpar_name):
            log.info("LPAR recovered at managed system %s" % src_mg_system)
            return True
        log.info("LPAR failed to recover at managed system %s" % src_mg_system)
        return False

    def get_adapter_id(self, mg_system, loc_code, remote_hmc=None):
        '''
        Get SRIOV adapter id

        :param mg_system: string, Source managed system
        :param loc_code: string, adapater
        :param remote_hmc: object, remote HMC instance
        :returns: string, adapter id
        '''
        hmc = remote_hmc if remote_hmc else self
        cmd = 'lshwres -m {} -r sriov --rsubtype adapter -F phys_loc:adapter_id'.format(
            mg_system)
        adapter_id_output = hmc.ssh.run_command(cmd)
        for line in adapter_id_output:
            if str(loc_code) in line:
                return line.split(':')[1]
        return ''

    def get_lpar_id(self, mg_system, l_lpar_name, remote_hmc=None):
        '''
        Get LPAR id

        :param mg_system: string, Source managed system
        :param l_lpar_name: string, LPAR name
        :param remote_hmc: object, remote HMC instance
        :returns: number, LPAR id on success, 0 on failure
        '''
        hmc = remote_hmc if remote_hmc else self
        cmd = 'lssyscfg -m %s -r lpar --filter lpar_names=%s -F lpar_id' % (
            mg_system, l_lpar_name)
        lpar_id_output = hmc.ssh.run_command(cmd)
        for line in lpar_id_output:
            if l_lpar_name in line:
                return 0
            return line
        return 0

    def is_msp_enabled(self, mg_system, vios_name, remote_hmc=None):
        '''
        The function checks if the moving service option is enabled
        on the given lpar partition.

        :param mg_system: string, Source managed system
        :param vios_name: string, VIOS name
        :param remote_hmc: object, remote HMC instance
        :returns: Boolean, True on success, False on failure
        '''
        hmc = remote_hmc if remote_hmc else self
        cmd = "lssyscfg -m %s -r lpar --filter lpar_names=%s -F msp" % (
            mg_system, vios_name)
        msp_output = hmc.ssh.run_command(cmd)
        if int(msp_output[0]) != 1:
            return False
        return True

    def gather_logs(self,
                    list_of_commands=[],
                    remote_hmc=None,
                    output_dir=None):
        '''
        Gather the logs for the commands at the given directory

        :param list_of_commands: list, list of commands
        :param remote_hmc: object, remote HMC instance
        :param output_dir: string, dir to store logs
        :returns: Boolean, True on success, False on failure
        :raises: `CommandFailed`
        '''
        hmc = remote_hmc if remote_hmc else self
        if not output_dir:
            output_dir = "HMC_Logs_%s" % (time.asctime(
                time.localtime())).replace(" ", "_")
        output_dir = os.path.join(self.system.cv_HOST.results_dir, output_dir,
                                  "hmc-" + hmc.hmc_ip)
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)

        default_commands = ['lshmc -V', 'pedbg -c -q 4']
        list_of_commands.extend(default_commands)

        try:
            for cmd in set(list_of_commands):
                output = "\n".join(hmc.run_command(r"%s" % cmd, timeout=1800))
                filename = "%s.log" % '-'.join(
                    (re.sub(r'[^a-zA-Z0-9]', ' ', cmd)).split())
                filepath = os.path.join(output_dir, filename)
                with open(filepath, 'w') as f:
                    f.write(output)
            log.warn(
                "Please collect the pedbg logs immediately. At risk of being overwritten."
            )
            return True
        except CommandFailed as cmd_failed:
            raise cmd_failed

    def run_command_ignore_fail(self, command, timeout=60, retry=0):
        '''
        Wrapper function for `ssh.run_command_ignore_fail`

        :param command: string, command
        :param timeout: number, time out in seconds
        :param retry: number, number of retries
        '''
        return self.ssh.run_command_ignore_fail(command,
                                                timeout * self.timeout_factor,
                                                retry)

    def run_command(self, i_cmd, timeout=15):
        '''
        Wrapper function for `ssh.run_command`

        :param i_cmd: string, command
        :param timeout: number, time out in seconds
        '''
        return self.ssh.run_command(i_cmd, timeout)