def __init__(self): # initialize OpTestUtil with this object the OpTestConfiguration self.util = OpTestUtil(self) # initialize OpTestCronus with this object the OpTestConfiguration self.cronus = OpTestCronus(self) self.args = [] self.remaining_args = [] self.basedir = os.path.dirname(sys.argv[0]) self.signal_ready = False # indicator for properly initialized self.atexit_ready = False # indicator for properly initialized self.aes_print_helpers = True # Need state for locker_wait self.dump = True # Need state for cleanup self.lock_dict = {'res_id': None, 'name': None, 'Group_Name': None, 'envs': [], } self.util_server = None # Hostlocker or AES self.util_bmc_server = None # OpenBMC REST Server atexit.register(self.cleanup) # allows cleanup handler to run (OpExit) self.firmware_versions = None self.nvram_debug_opts = None for dirname in (next(os.walk(os.path.join(self.basedir, 'addons')))[1]): if dirname == "__pycache__": continue optAddons[dirname] = importlib.import_module( "addons." + dirname + ".OpTest" + dirname + "Setup") return
def setUp(self): conf = OpTestConfiguration.conf self.cv_HOST = conf.host() self.cv_IPMI = conf.ipmi() self.cv_FSP = conf.bmc() self.cv_SYSTEM = conf.system() self.bmc_type = conf.args.bmc_type self.util = OpTestUtil()
def __init__(self, hmc_ip, user_name, password, managed_system, lpar_name, lpar_vios, lpar_prof, lpar_user, lpar_password, block_setup_term=None, delaybeforesend=None, timeout_factor=1, logfile=sys.stdout, prompt=None, scratch_disk="", check_ssh_keys=False, known_hosts_file=None, proxy=""): self.logfile = logfile self.hmc_ip = hmc_ip self.user = user_name self.passwd = password self.mg_system = managed_system self.util = OpTestUtil() self.expect_prompt = self.util.build_prompt(prompt) + "$" self.lpar_name = lpar_name self.lpar_vios = lpar_vios self.lpar_prof = lpar_prof self.lpar_user = lpar_user self.lpar_password = lpar_password self.scratch_disk = scratch_disk self.proxy = proxy self.state = ConsoleState.DISCONNECTED self.delaybeforesend = delaybeforesend self.system = None # OpTestUtil instance is NOT conf's self.prompt = prompt self.pty = None self.delaybeforesend = delaybeforesend # allows caller specific control of when to block setup_term self.block_setup_term = block_setup_term # tells setup_term to not throw exceptions, like when system off self.setup_term_quiet = 0 # flags the object to abandon setup_term operations, like when system off self.setup_term_disable = 0 # FUTURE - System Console currently tracked in System Object # state tracking, reset on boot and state changes self.PS1_set = -1 self.LOGIN_set = -1 self.SUDO_set = -1 super(HMCConsole, self).__init__(hmc_ip, user_name, password, scratch_disk, proxy, logfile, managed_system, lpar_name, prompt, block_setup_term, delaybeforesend, timeout_factor, lpar_prof, lpar_vios, lpar_user, lpar_password, check_ssh_keys, known_hosts_file)
def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_hostip=None, i_hostuser=None, i_hostPasswd=None): self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir) self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostip, i_hostuser, i_hostPasswd) self.cv_HOST = OpTestHost(i_hostip, i_hostuser, i_hostPasswd, i_bmcIP) self.cv_SYSTEM = OpTestSystem(i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostip, i_hostuser, i_hostPasswd) self.util = OpTestUtil()
def setUp(self): conf = OpTestConfiguration.conf self.cv_SYSTEM = conf.system() self.cv_BMC = conf.bmc() self.cv_HOST = conf.host() self.cv_IPMI = conf.ipmi() self.platform = conf.platform() self.util = OpTestUtil() self.bmc_type = conf.args.bmc_type self.bmc_ip = conf.args.bmc_ip self.bmc_username = conf.args.bmc_username self.bmc_password = conf.args.bmc_password
def setUp(self): conf = OpTestConfiguration.conf self.cv_SYSTEM = conf.system() self.cv_BMC = conf.bmc() self.cv_REST = self.cv_BMC.get_rest_api() self.cv_HOST = conf.host() self.cv_IPMI = conf.ipmi() self.platform = conf.platform() self.util = OpTestUtil() self.bmc_type = conf.args.bmc_type self.bmc_ip = conf.args.bmc_ip self.bmc_username = conf.args.bmc_username self.bmc_password = conf.args.bmc_password self.pupdate_binary = conf.args.pupdate self.pflash = conf.args.pflash
def __init__(self, hmc_ip, user_name, password, scratch_disk="", proxy="", logfile=sys.stdout, managed_system=None, lpar_name=None, prompt=None, block_setup_term=None, delaybeforesend=None, timeout_factor=None, lpar_prof=None, lpar_vios=None, lpar_user=None, lpar_password=None, check_ssh_keys=False, known_hosts_file=None, tgt_managed_system=None, tgt_lpar=None): self.hmc_ip = hmc_ip self.user = user_name self.passwd = password self.logfile = logfile self.mg_system = managed_system self.tgt_mg_system = tgt_managed_system self.tgt_lpar = tgt_lpar self.check_ssh_keys = check_ssh_keys self.known_hosts_file = known_hosts_file self.lpar_name = lpar_name self.lpar_prof = lpar_prof self.lpar_user = lpar_user self.lpar_password = lpar_password self.lpar_vios = lpar_vios self.util = OpTestUtil() self.prompt = prompt self.expect_prompt = self.util.build_prompt(prompt) + "$" self.ssh = OpTestSSH(hmc_ip, user_name, password, logfile=self.logfile, check_ssh_keys=check_ssh_keys, known_hosts_file=known_hosts_file, block_setup_term=block_setup_term) self.scratch_disk = scratch_disk self.proxy = proxy self.scratch_disk_size = None self.delaybeforesend = delaybeforesend self.system = None # OpTestUtil instance is NOT conf's self.pty = None # allows caller specific control of when to block setup_term self.block_setup_term = block_setup_term # tells setup_term to not throw exceptions, like when system off self.setup_term_quiet = 0 # flags the object to abandon setup_term operations, like when system off self.setup_term_disable = 0 # functional simulators are very slow, so multiply all default timeouts by this factor self.timeout_factor = timeout_factor # state tracking, reset on boot and state changes # console tracking done on System object for the system console self.PS1_set = -1 self.LOGIN_set = -1 self.SUDO_set = -1
def __init__(self): self.util = OpTestUtil(self) # initialize OpTestUtil with this object the OpTestConfiguration self.cronus = OpTestCronus(self) # initialize OpTestCronus with this object the OpTestConfiguration self.args = [] self.remaining_args = [] self.basedir = os.path.dirname(sys.argv[0]) self.signal_ready = False # indicator for properly initialized self.atexit_ready = False # indicator for properly initialized self.aes_print_helpers = True # Need state for locker_wait self.dump = True # Need state for cleanup self.lock_dict = { 'res_id' : None, 'name' : None, 'Group_Name' : None, 'envs' : [], } self.util_server = None # Hostlocker or AES self.util_bmc_server = None # OpenBMC REST Server atexit.register(self.__del__) # allows cleanup handler to run (OpExit) self.firmware_versions = None for dir in (os.walk(os.path.join(self.basedir, 'addons')).next()[1]): optAddons[dir] = importlib.import_module("addons." + dir + ".OpTest" + dir + "Setup") return
def setUpClass(cls): conf = OpTestConfiguration.conf cls.cv_HOST = conf.host() cls.cv_IPMI = conf.ipmi() cls.cv_SYSTEM = conf.system() cls.util = OpTestUtil() cls.test = None
def setUp(self): conf = OpTestConfiguration.conf self.cv_HOST = conf.host() self.cv_IPMI = conf.ipmi() self.cv_SYSTEM = conf.system() self.util = OpTestUtil() self.ppc64cpu_freq_re = re.compile(r"([a-z]+):\s+([\d.]+)")
def setUp(self): conf = OpTestConfiguration.conf self.conf = conf self.cv_HOST = conf.host() self.cv_IPMI = conf.ipmi() self.cv_SYSTEM = conf.system() self.cv_BMC = conf.bmc() self.util = OpTestUtil() self.bmc_type = conf.args.bmc_type if not (self.conf.args.os_repo or self.conf.args.os_cdrom): self.fail( "Provide installation media for installation with --os-repo or --os-cdrom" ) if "qemu" not in self.bmc_type and not ( self.conf.args.host_ip and self.conf.args.host_gateway and self.conf.args.host_dns and self.conf.args.host_submask and self.conf.args.host_mac): self.fail( "Provide host network details refer, --host-{ip,gateway,dns,submask,mac}" ) if not (self.conf.args.host_user and self.conf.args.host_password): self.fail( "Provide host user details refer, --host-{user,password}") if not self.cv_HOST.get_scratch_disk(): self.fail( "Provide proper host disk to install refer, --host-scratch-disk" )
def setUp(self): conf = OpTestConfiguration.conf self.cv_HOST = conf.host() self.cv_IPMI = conf.ipmi() self.cv_SYSTEM = conf.system() self.util = OpTestUtil() self.platform = conf.platform()
def setUp(self): conf = OpTestConfiguration.conf self.host = conf.host() self.ipmi = conf.ipmi() self.system = conf.system() self.util = OpTestUtil() pass
def setUp(self): self.conf = OpTestConfiguration.conf self.host = self.conf.host() self.ipmi = self.conf.ipmi() self.system = self.conf.system() self.bmc = self.conf.bmc() self.util = OpTestUtil() self.bmc_type = self.conf.args.bmc_type if not (self.conf.args.os_repo or self.conf.args.os_cdrom): self.fail( "Provide installation media for installation, --os-repo is missing" ) if not (self.conf.args.host_ip and self.conf.args.host_gateway and self.conf.args.host_dns and self.conf.args.host_submask and self.conf.args.host_mac): self.fail( "Provide host network details refer, --host-{ip,gateway,dns,submask,mac}" ) if not (self.conf.args.host_user and self.conf.args.host_password): self.fail( "Provide host user details refer, --host-{user,password}") if not self.conf.args.host_scratch_disk: self.fail( "Provide proper host disk to install refer, --host-scratch-disk" )
def setUp(self): conf = OpTestConfiguration.conf self.host = conf.host() self.ipmi = conf.ipmi() self.system = conf.system() self.bmc = conf.bmc() self.util = OpTestUtil() self.bmc_type = conf.args.bmc_type
def setUp(self): conf = OpTestConfiguration.conf self.cv_HOST = conf.host() self.cv_IPMI = conf.ipmi() self.cv_SYSTEM = conf.system() self.cv_BMC = conf.bmc() self.util = OpTestUtil() self.pci_good_data_file = conf.lspci_file()
class OpTestFlashBase(unittest.TestCase): def setUp(self): conf = OpTestConfiguration.conf self.cv_SYSTEM = conf.system() self.cv_BMC = conf.bmc() self.cv_HOST = conf.host() self.cv_IPMI = conf.ipmi() self.platform = conf.platform() self.util = OpTestUtil() self.bmc_type = conf.args.bmc_type self.bmc_ip = conf.args.bmc_ip self.bmc_username = conf.args.bmc_username self.bmc_password = conf.args.bmc_password def validate_side_activated(self): l_bmc_side, l_pnor_side = self.cv_IPMI.ipmi_get_side_activated() self.assertIn(BMC_CONST.PRIMARY_SIDE, l_bmc_side, "BMC: Primary side is not active") # TODO force setting of primary side to BIOS Golden side sensor self.assertIn(BMC_CONST.PRIMARY_SIDE, l_pnor_side, "PNOR: Primary side is not active") def get_pnor_level(self): rc = self.cv_IPMI.ipmi_get_PNOR_level() print rc def bmc_down_check(self): cmd = "ping -c 1 " + self.cv_BMC.host_name + " 1> /dev/null; echo $?" count = 0 while count < 500: output = commands.getstatusoutput(cmd) if output[1] != '0': print "FSP/BMC Comes down" break count = count + 1 time.sleep(2) else: self.assertTrue(False, "FSP/BMC keeps on pinging up") return True def scp_file(self, src_file_path, dst_file_path): self.util.copyFilesToDest(src_file_path, self.bmc_username, self.bmc_ip, dst_file_path, self.bmc_password, "2", BMC_CONST.SCP_TO_REMOTE)
def setUp(self): conf = OpTestConfiguration.conf self.host = conf.host() self.ipmi = conf.ipmi() self.system = conf.system() self.bmc = conf.bmc() self.util = OpTestUtil() self.pci_good_data_file = conf.lspci_file()
def setUp(self): conf = OpTestConfiguration.conf self.cv_HOST = conf.host() self.cv_IPMI = conf.ipmi() self.cv_SYSTEM = conf.system() self.cv_BMC = conf.bmc() self.util = OpTestUtil() pass
def setUpClass(cls): conf = OpTestConfiguration.conf cls.cv_HOST = conf.host() cls.cv_IPMI = conf.ipmi() cls.cv_FSP = conf.bmc() cls.cv_SYSTEM = conf.system() cls.bmc_type = conf.args.bmc_type cls.util = OpTestUtil()
def setUpClass(cls): conf = OpTestConfiguration.conf cls.cv_HOST = conf.host() cls.cv_IPMI = conf.ipmi() cls.cv_SYSTEM = conf.system() cls.util = OpTestUtil() # By default test will run on all PHBs/PE's, if one want to skip certain ones mention in this format. cls.skip_phbs = [] # ['PCI0001', 'PCI0002', 'PCI0003', 'PCI0004', 'PCI0005', 'PCI0030', 'PCI0031', 'PCI0032'] cls.skip_pes = [] # ['0002:00:00.0']
def __init__(self, hmc_ip, user_name, password, scratch_disk="", proxy="", logfile=sys.stdout, managed_system=None, lpar_name=None, prompt=None, lpar_prof=None, lpar_vios=None, lpar_user=None, lpar_password=None, check_ssh_keys=False, known_hosts_file=None): self.hmc_ip = hmc_ip self.user = user_name self.passwd = password self.logfile = logfile self.system = managed_system self.system = managed_system self.check_ssh_keys = check_ssh_keys self.known_hosts_file = known_hosts_file self.lpar_name = lpar_name self.lpar_prof = lpar_prof self.lpar_user = lpar_user self.lpar_password = lpar_password self.lpar_vios = lpar_vios self.lpar_con = None self.vterm = False self.util = OpTestUtil() self.prompt = prompt self.expect_prompt = self.util.build_prompt(prompt) + "$" self.conf = OpTestConfiguration.conf self.ssh = OpTestSSH(hmc_ip, user_name, password, logfile=self.logfile, check_ssh_keys=check_ssh_keys, known_hosts_file=known_hosts_file) self.scratch_disk = scratch_disk self.proxy = proxy self.scratch_disk_size = None
def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_hostip=None, i_hostuser=None, i_hostPasswd=None): self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir) self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir) self.cv_HOST = OpTestHost(i_hostip, i_hostuser, i_hostPasswd, i_bmcIP) self.util = OpTestUtil() self.host_user = i_hostuser self.host_ip = i_hostip self.host_Passwd = i_hostPasswd
def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_lparip=None, i_lparuser=None, i_lparPasswd=None): self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir) self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir) self.cv_LPAR = OpTestLpar(i_lparip, i_lparuser, i_lparPasswd, i_bmcIP) self.util = OpTestUtil() self.lpar_user = i_lparuser self.lpar_ip = i_lparip self.lpar_Passwd = i_lparPasswd
def __init__(self): self.util = OpTestUtil(self) # initialize OpTestUtil with this object the OpTestConfiguration self.args = [] self.remaining_args = [] self.basedir = os.path.dirname(sys.argv[0]) self.signal_ready = False # indicator for properly initialized self.lock_dict = { 'res_id' : None, 'name' : None, 'Group_Name' : None, 'envs' : [], } self.util_server = None for dir in (os.walk(os.path.join(self.basedir, 'addons')).next()[1]): optAddons[dir] = importlib.import_module("addons." + dir + ".OpTest" + dir + "Setup") return
class ControlC(unittest.TestCase): ''' Start a process that does a bunch of console output, and then try and 'control-c' it to stop the process and get a prompt back. ''' CONTROL = 'c' def setUp(self): conf = OpTestConfiguration.conf self.cv_BMC = conf.bmc() self.cv_SYSTEM = conf.system() self.util = OpTestUtil() self.prompt = self.util.build_prompt() def cleanup(self): pass def runTest(self): self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL) console = self.cv_BMC.get_host_console() if (isinstance(self.cv_BMC, OpTestMambo.OpTestMambo)): raise unittest.SkipTest("Mambo so skipping Control-C tests") raw_pty = console.get_console() raw_pty.sendline("find /") time.sleep(2) raw_pty.sendcontrol(self.CONTROL) BMC_DISCONNECT = 'SOL session closed by BMC' timeout = 60 try: rc = raw_pty.expect([BMC_DISCONNECT, self.prompt, pexpect.TIMEOUT, pexpect.EOF], timeout) if rc == 0: raise BMCDisconnected(BMC_DISCONNECT) self.assertEqual(rc, 1, "Failed to find expected prompt") except pexpect.TIMEOUT as e: log.debug(e) log.debug("# TIMEOUT waiting for command to finish with ctrl-c.") log.debug("# Everything is terrible. Fail the world, " "power cycle (if lucky)") self.cv_SYSTEM.set_state(OpSystemState.UNKNOWN_BAD) self.fail("Could not ctrl-c running command in reasonable time") self.cleanup()
class ControlC(unittest.TestCase): CONTROL = 'c' def setUp(self): conf = OpTestConfiguration.conf self.cv_BMC = conf.bmc() self.cv_SYSTEM = conf.system() self.util = OpTestUtil() self.prompt = self.util.build_prompt() def cleanup(self): pass def runTest(self): self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL) console = self.cv_BMC.get_host_console() # I should really make this API less nasty... raw_console = console.get_console() #raw_console.sendline("hexdump -C -v /dev/zero") raw_console.sendline("find /") time.sleep(2) raw_console.sendcontrol(self.CONTROL) BMC_DISCONNECT = 'SOL session closed by BMC' timeout = 60 try: rc = raw_console.expect( [BMC_DISCONNECT, self.prompt, pexpect.TIMEOUT, pexpect.EOF], timeout) if rc == 0: raise BMCDisconnected(BMC_DISCONNECT) self.assertEqual(rc, 1, "Failed to find expected prompt") except pexpect.TIMEOUT as e: log.debug(e) log.debug("# TIMEOUT waiting for command to finish with ctrl-c.") log.debug( "# Everything is terrible. Fail the world, power cycle (if lucky)" ) self.cv_SYSTEM.set_state(OpSystemState.UNKNOWN_BAD) self.fail("Could not ctrl-c running command in reasonable time") self.cleanup()
class OpTestConfiguration(): def __init__(self): self.util = OpTestUtil(self) # initialize OpTestUtil with this object the OpTestConfiguration self.args = [] self.remaining_args = [] self.basedir = os.path.dirname(sys.argv[0]) self.signal_ready = False # indicator for properly initialized self.atexit_ready = False # indicator for properly initialized self.aes_print_helpers = True # Need state for locker_wait self.dump = True # Need state for cleanup self.lock_dict = { 'res_id' : None, 'name' : None, 'Group_Name' : None, 'envs' : [], } self.util_server = None # Hostlocker or AES self.util_bmc_server = None # OpenBMC REST Server atexit.register(self.__del__) # allows cleanup handler to run (OpExit) self.firmware_versions = None for dir in (os.walk(os.path.join(self.basedir, 'addons')).next()[1]): optAddons[dir] = importlib.import_module("addons." + dir + ".OpTest" + dir + "Setup") return def __del__(self): if self.atexit_ready: # calling cleanup before args initialized pointless # attribute errors thrown in cleanup, e.g. ./op-test -h self.util.cleanup() def parse_args(self, argv=None): conf_parser = argparse.ArgumentParser(add_help=False) # We have two parsers so we have correct --help, we need -c in both conf_parser.add_argument("-c", "--config-file", help="Configuration File", metavar="FILE") args , remaining_args = conf_parser.parse_known_args(argv) defaults = {} config = ConfigParser.SafeConfigParser() config.read([os.path.expanduser("~/.op-test-framework.conf")]) if args.config_file: if os.access(args.config_file, os.R_OK): config.read([args.config_file]) else: raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), args.config_file) try: defaults = dict(config.items('op-test')) except ConfigParser.NoSectionError: pass parser = get_parser() parser.set_defaults(**defaults) if defaults.get('qemu_binary'): qemu_default = defaults['qemu_binary'] if defaults.get('mambo_binary'): mambo_default = defaults['mambo_binary'] if defaults.get('mambo_initial_run_script'): mambo_default = defaults['mambo_initial_run_script'] parser.add_argument("--check-ssh-keys", action='store_true', default=False, help="Check remote host keys when using SSH (auto-yes on new)") parser.add_argument("--known-hosts-file", help="Specify a custom known_hosts file") self.args , self.remaining_args = parser.parse_known_args(remaining_args) args_dict = vars(self.args) # if we have a bmc_type we start with appropriate template if args_dict.get('bmc_type') is not None: dict_merge = default_templates.get(args_dict.get('bmc_type').lower()) if dict_merge is not None: default_val.update(dict_merge) # overlays dict merge on top of default_val for key in default_val: if args_dict.get(key) is None: args_dict[key] = default_val[key] stateMap = { 'UNKNOWN' : common.OpTestSystem.OpSystemState.UNKNOWN, 'UNKNOWN_BAD' : common.OpTestSystem.OpSystemState.UNKNOWN_BAD, 'OFF' : common.OpTestSystem.OpSystemState.OFF, 'PETITBOOT' : common.OpTestSystem.OpSystemState.PETITBOOT, 'PETITBOOT_SHELL' : common.OpTestSystem.OpSystemState.PETITBOOT_SHELL, 'OS' : common.OpTestSystem.OpSystemState.OS } # Some quick sanity checking if self.args.known_hosts_file and not self.args.check_ssh_keys: parser.error("--known-hosts-file requires --check-ssh-keys") # Setup some defaults for the output options # Order of precedence # 1. cmdline arg # 2. env variable # 3. default path if (self.args.output): outdir = self.args.output elif ("OP_TEST_OUTPUT" in os.environ): outdir = os.environ["OP_TEST_OUTPUT"] else: outdir = os.path.join(self.basedir, "test-reports") self.outsuffix = "test-run-%s" % self.get_suffix() outdir = os.path.join(outdir, self.outsuffix) # Normalize the path to fully qualified and create if not there self.output = os.path.abspath(outdir) if (not os.path.exists(self.output)): os.makedirs(self.output) if (self.args.logdir): logdir = self.args.logdir elif ("OP_TEST_LOGDIR" in os.environ): logdir = os.environ["OP_TEST_LOGDIR"] else: logdir = self.output self.logdir = os.path.abspath(logdir) if (not os.path.exists(self.logdir)): os.makedirs(self.logdir) print("Logs in: {}".format(self.logdir)) OpTestLogger.optest_logger_glob.logdir = self.logdir # Grab the suffix, if not given use current time self.outsuffix = self.get_suffix() # set up where all the logs go logfile = os.path.join(self.output, "%s.log" % self.outsuffix) logcmd = "tee %s" % (logfile) # we use 'cat -v' to convert control characters # to something that won't affect the user's terminal if self.args.quiet: logcmd = logcmd + "> /dev/null" # save sh_level for later refresh loggers OpTestLogger.optest_logger_glob.sh_level = logging.ERROR OpTestLogger.optest_logger_glob.sh.setLevel(logging.ERROR) else: logcmd = logcmd + "| sed -u -e 's/\\r$//g'|cat -v" # save sh_level for later refresh loggers OpTestLogger.optest_logger_glob.sh_level = logging.INFO OpTestLogger.optest_logger_glob.sh.setLevel(logging.INFO) OpTestLogger.optest_logger_glob.setUpLoggerFile(datetime.utcnow().strftime("%Y%m%d%H%M%S%f")+'.main.log') OpTestLogger.optest_logger_glob.setUpLoggerDebugFile(datetime.utcnow().strftime("%Y%m%d%H%M%S%f")+'.debug.log') OpTestLogger.optest_logger_glob.optest_logger.info('TestCase Log files: {}/*{}*'.format(self.output, self.outsuffix)) OpTestLogger.optest_logger_glob.optest_logger.info('StreamHandler setup {}'.format('quiet' if self.args.quiet else 'normal')) self.logfile_proc = subprocess.Popen(logcmd, stdin=subprocess.PIPE, stderr=sys.stderr, stdout=sys.stdout, shell=True) self.logfile = self.logfile_proc.stdin # we have enough setup to allow # signal handler cleanup to run self.signal_ready = True # atexit viable for cleanup to run self.atexit_ready = True # now that we have loggers, dump conf file to help debug later OpTestLogger.optest_logger_glob.optest_logger.debug( "conf file defaults={}".format(defaults)) # setup AES and Hostlocker configs after the logging is setup locker_timeout = time.time() + 60*self.args.locker_wait locker_code = errno.ETIME # 62 locker_message = ("OpTestSystem waited {} minutes but was unable" " to lock environment/host requested," " either pick another environment/host or increase " "--locker-wait, try --aes q with options for " "--aes-search-args to view availability, or as" " appropriate for your hostlocker" .format(self.args.locker_wait)) locker_exit_exception = OpExit(message=locker_message, code=locker_code) while True: try: rollup_flag = False self.util.check_lockers() break except Exception as e: OpTestLogger.optest_logger_glob.optest_logger.debug( "locker_wait Exception={}".format(e)) if "unable to lock" in e.message: self.aes_print_helpers = False # SystemExit exception needs message to print rollup_message = locker_exit_exception.message rollup_exception = locker_exit_exception else: rollup_message = e.message rollup_exception = e rollup_flag = True # bubble exception out if time.time() > locker_timeout or rollup_flag: # if not "unable to lock" we bubble up underlying exception OpTestLogger.optest_logger_glob.optest_logger.warning( "{}".format(rollup_message)) raise rollup_exception else: OpTestLogger.optest_logger_glob.optest_logger.info( "OpTestSystem waiting for requested environment/host" " total time to wait is {} minutes, we will check" " every minute" .format(self.args.locker_wait)) time.sleep(60) if self.args.machine_state == None: if self.args.bmc_type in ['qemu', 'mambo']: # Force UNKNOWN_BAD so that we don't try to setup the console early self.startState = common.OpTestSystem.OpSystemState.UNKNOWN_BAD else: self.startState = common.OpTestSystem.OpSystemState.UNKNOWN else: self.startState = stateMap[self.args.machine_state] return self.args, self.remaining_args def get_suffix(self): # Grab the suffix, if not given use current time if (self.args.suffix): outsuffix = self.args.suffix else: outsuffix = time.strftime("%Y%m%d%H%M%S") return outsuffix def objs(self): if self.args.list_suites or self.args.list_tests: return # check to see if bmc_ip even pings to validate configuration parms try: self.util.PingFunc(self.args.bmc_ip, totalSleepTime=BMC_CONST.PING_RETRY_FOR_STABILITY) except Exception as e: # we are trying to catch sooner rather than later # if we have reservations that need cleaned up # otherwise we would have to try/except for cleanup # in lots of places # testcases.HelloWorld in CI fails if we throw this # raise only if we have reservations to cleanup if self.args.hostlocker is not None \ or self.args.aes is not None \ or self.args.aes_search_args is not None: self.util.cleanup() raise ParameterCheck(message="OpTestSystem PingFunc fails to " "ping '{}', check your configuration and setup, see " "Exception details: {}".format(self.args.bmc_ip, e)) try: host = common.OpTestHost.OpTestHost(self.args.host_ip, self.args.host_user, self.args.host_password, self.args.bmc_ip, self.output, scratch_disk=self.args.host_scratch_disk, proxy=self.args.proxy, check_ssh_keys=self.args.check_ssh_keys, known_hosts_file=self.args.known_hosts_file, conf=self) if self.args.bmc_type in ['AMI', 'SMC']: web = OpTestWeb(self.args.bmc_ip, self.args.bmc_usernameipmi, self.args.bmc_passwordipmi) bmc = None if self.args.bmc_type in ['AMI']: ipmi = OpTestIPMI(self.args.bmc_ip, self.args.bmc_usernameipmi, self.args.bmc_passwordipmi, host=host, logfile=self.logfile, ) bmc = OpTestBMC(ip=self.args.bmc_ip, username=self.args.bmc_username, password=self.args.bmc_password, logfile=self.logfile, ipmi=ipmi, web=web, check_ssh_keys=self.args.check_ssh_keys, known_hosts_file=self.args.known_hosts_file ) elif self.args.bmc_type in ['SMC']: ipmi = OpTestSMCIPMI(self.args.bmc_ip, self.args.bmc_usernameipmi, self.args.bmc_passwordipmi, logfile=self.logfile, host=host, ) bmc = OpTestSMC(ip=self.args.bmc_ip, username=self.args.bmc_username, password=self.args.bmc_password, ipmi=ipmi, web=web, check_ssh_keys=self.args.check_ssh_keys, known_hosts_file=self.args.known_hosts_file ) self.op_system = common.OpTestSystem.OpTestSystem( state=self.startState, bmc=bmc, host=host, conf=self, ) ipmi.set_system(self.op_system) bmc.set_system(self.op_system) elif self.args.bmc_type in ['FSP']: ipmi = OpTestIPMI(self.args.bmc_ip, None, # FSP does not use UID self.args.bmc_passwordipmi, host=host, logfile=self.logfile) bmc = OpTestFSP(self.args.bmc_ip, self.args.bmc_username, self.args.bmc_password, ipmi=ipmi, ) self.op_system = common.OpTestSystem.OpTestFSPSystem( state=self.startState, bmc=bmc, host=host, conf=self, ) ipmi.set_system(self.op_system) elif self.args.bmc_type in ['OpenBMC']: ipmi = OpTestIPMI(self.args.bmc_ip, self.args.bmc_usernameipmi, self.args.bmc_passwordipmi, host=host, logfile=self.logfile) rest_api = HostManagement(conf=self, ip=self.args.bmc_ip, username=self.args.bmc_username, password=self.args.bmc_password) bmc = OpTestOpenBMC(ip=self.args.bmc_ip, username=self.args.bmc_username, password=self.args.bmc_password, ipmi=ipmi, rest_api=rest_api, logfile=self.logfile, check_ssh_keys=self.args.check_ssh_keys, known_hosts_file=self.args.known_hosts_file) self.op_system = common.OpTestSystem.OpTestOpenBMCSystem( host=host, bmc=bmc, state=self.startState, conf=self, ) bmc.set_system(self.op_system) elif self.args.bmc_type in ['qemu']: print(repr(self.args)) bmc = OpTestQemu(qemu_binary=self.args.qemu_binary, pnor=self.args.host_pnor, skiboot=self.args.flash_skiboot, kernel=self.args.flash_kernel, initramfs=self.args.flash_initramfs, cdrom=self.args.os_cdrom, logfile=self.logfile, hda=self.args.host_scratch_disk) self.op_system = common.OpTestSystem.OpTestQemuSystem(host=host, bmc=bmc, state=self.startState, conf=self, ) bmc.set_system(self.op_system) elif self.args.bmc_type in ['mambo']: if not (os.stat(self.args.mambo_binary).st_mode & stat.S_IXOTH): raise ParameterCheck(message="Check that the file exists with" " X permissions" " mambo-binary={}" .format(self.args.mambo_binary)) if self.args.flash_skiboot is None \ or not os.access(self.args.flash_skiboot, os.R_OK|os.W_OK): raise ParameterCheck(message="Check that the file exists with" " R/W/X permissions" " flash-skiboot={}" .format(self.args.flash_skiboot)) if self.args.flash_kernel is None \ or not os.access(self.args.flash_kernel, os.R_OK|os.W_OK): raise ParameterCheck(message="Check that the file exists with" " R/W permissions" " flash-kernel={}" .format(self.args.flash_kernel)) bmc = OpTestMambo(mambo_binary=self.args.mambo_binary, mambo_initial_run_script=self.args.mambo_initial_run_script, mambo_autorun=self.args.mambo_autorun, skiboot=self.args.flash_skiboot, kernel=self.args.flash_kernel, initramfs=self.args.flash_initramfs, logfile=self.logfile) self.op_system = common.OpTestSystem.OpTestMamboSystem(host=host, bmc=bmc, state=self.startState, conf=self, ) bmc.set_system(self.op_system) # Check that the bmc_type exists in our loaded addons then create our objects elif self.args.bmc_type in optAddons: (bmc, self.op_system) = optAddons[self.args.bmc_type].createSystem(self, host) else: self.util.cleanup() raise Exception("Unsupported BMC Type '{}', check your " "upper/lower cases for bmc_type and verify " "any credentials used from HostLocker or " "AES Version (see aes_get_creds " "version_mappings)".format(self.args.bmc_type)) host.set_system(self.op_system) return except Exception as e: traceback.print_exc() self.util.cleanup() raise e def bmc(self): return self.op_system.bmc def system(self): return self.op_system def host(self): return self.op_system.host() def ipmi(self): return self.op_system.ipmi() def lspci_file(self): return self.args.host_lspci def platform(self): return self.args.platform
class OpTestConfiguration(): def __init__(self): # initialize OpTestUtil with this object the OpTestConfiguration self.util = OpTestUtil(self) # initialize OpTestCronus with this object the OpTestConfiguration self.cronus = OpTestCronus(self) self.args = [] self.remaining_args = [] self.basedir = os.path.dirname(sys.argv[0]) self.signal_ready = False # indicator for properly initialized self.atexit_ready = False # indicator for properly initialized self.aes_print_helpers = True # Need state for locker_wait self.dump = True # Need state for cleanup self.lock_dict = {'res_id': None, 'name': None, 'Group_Name': None, 'envs': [], } self.util_server = None # Hostlocker or AES self.util_bmc_server = None # OpenBMC REST Server atexit.register(self.cleanup) # allows cleanup handler to run (OpExit) self.firmware_versions = None self.nvram_debug_opts = None for dirname in (next(os.walk(os.path.join(self.basedir, 'addons')))[1]): if dirname == "__pycache__": continue optAddons[dirname] = importlib.import_module( "addons." + dirname + ".OpTest" + dirname + "Setup") return def cleanup(self): if self.atexit_ready: # calling cleanup before args initialized pointless # attribute errors thrown in cleanup, e.g. ./op-test -h self.util.cleanup() def parse_args(self, argv=None): conf_parser = argparse.ArgumentParser(add_help=False) # We have two parsers so we have correct --help, we need -c in both conf_parser.add_argument("-c", "--config-file", help="Configuration File", metavar="FILE") args, remaining_args = conf_parser.parse_known_args(argv) defaults = {} config = configparser.SafeConfigParser() config.read([os.path.expanduser("~/.op-test-framework.conf")]) if args.config_file: if os.access(args.config_file, os.R_OK): config.read([args.config_file]) else: raise OSError(errno.ENOENT, os.strerror( errno.ENOENT), args.config_file) try: defaults = dict(config.items('op-test')) except configparser.NoSectionError: pass parser = get_parser() parser.set_defaults(**defaults) if defaults.get('qemu_binary'): qemu_default = defaults['qemu_binary'] if defaults.get('mambo_binary'): mambo_default = defaults['mambo_binary'] if defaults.get('mambo_initial_run_script'): mambo_default = defaults['mambo_initial_run_script'] parser.add_argument("--check-ssh-keys", action='store_true', default=False, help="Check remote host keys when using SSH (auto-yes on new)") parser.add_argument("--known-hosts-file", help="Specify a custom known_hosts file") self.args, self.remaining_args = parser.parse_known_args( remaining_args) args_dict = vars(self.args) # if we have a bmc_type we start with appropriate template if args_dict.get('bmc_type') is not None: dict_merge = default_templates.get( args_dict.get('bmc_type').lower()) if dict_merge is not None: # overlays dict merge on top of default_val default_val.update(dict_merge) for key in default_val: if args_dict.get(key) is None: args_dict[key] = default_val[key] stateMap = {'UNKNOWN': common.OpTestSystem.OpSystemState.UNKNOWN, 'UNKNOWN_BAD': common.OpTestSystem.OpSystemState.UNKNOWN_BAD, 'OFF': common.OpTestSystem.OpSystemState.OFF, 'PETITBOOT': common.OpTestSystem.OpSystemState.PETITBOOT, 'PETITBOOT_SHELL': common.OpTestSystem.OpSystemState.PETITBOOT_SHELL, 'OS': common.OpTestSystem.OpSystemState.OS } # Some quick sanity checking if self.args.known_hosts_file and not self.args.check_ssh_keys: parser.error("--known-hosts-file requires --check-ssh-keys") # Setup some defaults for the output options # Order of precedence # 1. cmdline arg # 2. env variable # 3. default path if (self.args.output): outdir = self.args.output elif ("OP_TEST_OUTPUT" in os.environ): outdir = os.environ["OP_TEST_OUTPUT"] else: outdir = os.path.join(self.basedir, "test-reports") self.outsuffix = "test-run-%s" % self.get_suffix() outdir = os.path.join(outdir, self.outsuffix) # Normalize the path to fully qualified and create if not there self.output = os.path.abspath(outdir) if (not os.path.exists(self.output)): os.makedirs(self.output) if (self.args.logdir): logdir = self.args.logdir elif ("OP_TEST_LOGDIR" in os.environ): logdir = os.environ["OP_TEST_LOGDIR"] else: logdir = self.output self.logdir = os.path.abspath(logdir) if (not os.path.exists(self.logdir)): os.makedirs(self.logdir) print(("Logs in: {}".format(self.logdir))) OpTestLogger.optest_logger_glob.logdir = self.logdir # Grab the suffix, if not given use current time self.outsuffix = self.get_suffix() # set up where all the logs go logfile = os.path.join(self.output, "%s.log" % self.outsuffix) logcmd = "tee %s" % (logfile) # we use 'cat -v' to convert control characters # to something that won't affect the user's terminal if self.args.quiet: logcmd = logcmd + "> /dev/null" # save sh_level for later refresh loggers OpTestLogger.optest_logger_glob.sh_level = logging.ERROR OpTestLogger.optest_logger_glob.sh.setLevel(logging.ERROR) else: logcmd = logcmd + "| sed -u -e 's/\\r$//g'|cat -v" # save sh_level for later refresh loggers OpTestLogger.optest_logger_glob.sh_level = logging.INFO OpTestLogger.optest_logger_glob.sh.setLevel(logging.INFO) OpTestLogger.optest_logger_glob.setUpLoggerFile( datetime.utcnow().strftime("%Y%m%d%H%M%S%f")+'.main.log') OpTestLogger.optest_logger_glob.setUpLoggerDebugFile( datetime.utcnow().strftime("%Y%m%d%H%M%S%f")+'.debug.log') OpTestLogger.optest_logger_glob.optest_logger.info( 'TestCase Log files: {}/*'.format(self.output)) OpTestLogger.optest_logger_glob.optest_logger.info( 'StreamHandler setup {}'.format('quiet' if self.args.quiet else 'normal')) self.logfile_proc = subprocess.Popen(logcmd, stdin=subprocess.PIPE, stderr=sys.stderr, stdout=sys.stdout, shell=True, universal_newlines=True, encoding='utf-8') self.logfile = self.logfile_proc.stdin # we have enough setup to allow # signal handler cleanup to run self.signal_ready = True # atexit viable for cleanup to run self.atexit_ready = True # now that we have loggers, dump conf file to help debug later OpTestLogger.optest_logger_glob.optest_logger.debug( "conf file defaults={}".format(defaults)) cmd = "git describe --always" try: git_output = subprocess.check_output(cmd.split()) # log for triage of how dated the repo is OpTestLogger.optest_logger_glob.optest_logger.debug( "op-test-framework git level = {}".format(git_output)) except Exception as e: OpTestLogger.optest_logger_glob.optest_logger.debug( "Unable to get git describe") # setup AES and Hostlocker configs after the logging is setup locker_timeout = time.time() + 60*self.args.locker_wait locker_code = errno.ETIME # 62 locker_message = ("OpTestSystem waited {} minutes but was unable" " to lock environment/host requested," " either pick another environment/host or increase " "--locker-wait, try --aes q with options for " "--aes-search-args to view availability, or as" " appropriate for your hostlocker" .format(self.args.locker_wait)) locker_exit_exception = OpExit(message=locker_message, code=locker_code) while True: try: rollup_flag = False self.util.check_lockers() break except Exception as e: OpTestLogger.optest_logger_glob.optest_logger.debug( "locker_wait Exception={}".format(e)) if "unable to lock" in e.message: self.aes_print_helpers = False # SystemExit exception needs message to print rollup_message = locker_exit_exception.message rollup_exception = locker_exit_exception else: rollup_message = e.message rollup_exception = e rollup_flag = True # bubble exception out if time.time() > locker_timeout or rollup_flag: # if not "unable to lock" we bubble up underlying exception OpTestLogger.optest_logger_glob.optest_logger.warning( "{}".format(rollup_message)) raise rollup_exception else: OpTestLogger.optest_logger_glob.optest_logger.info( "OpTestSystem waiting for requested environment/host" " total time to wait is {} minutes, we will check" " every minute" .format(self.args.locker_wait)) time.sleep(60) if self.args.machine_state == None: if self.args.bmc_type in ['qemu', 'mambo']: # Force UNKNOWN_BAD so that we don't try to setup the console early self.startState = common.OpTestSystem.OpSystemState.UNKNOWN_BAD else: self.startState = common.OpTestSystem.OpSystemState.UNKNOWN else: self.startState = stateMap[self.args.machine_state] return self.args, self.remaining_args def get_suffix(self): # Grab the suffix, if not given use current time if (self.args.suffix): outsuffix = self.args.suffix else: outsuffix = time.strftime("%Y%m%d%H%M%S") return outsuffix def objs(self): if self.args.list_suites or self.args.list_tests: return # check to see if bmc_ip even pings to validate configuration parms try: self.util.PingFunc( self.args.bmc_ip, totalSleepTime=BMC_CONST.PING_RETRY_FOR_STABILITY) except Exception as e: # we are trying to catch sooner rather than later # if we have reservations that need cleaned up # otherwise we would have to try/except for cleanup # in lots of places # testcases.HelloWorld in CI fails if we throw this # raise only if we have reservations to cleanup if self.args.hostlocker is not None \ or self.args.aes is not None \ or self.args.aes_search_args is not None: self.util.cleanup() raise ParameterCheck(message="OpTestSystem PingFunc fails to " "ping '{}', check your configuration and setup, see " "Exception details: {}".format(self.args.bmc_ip, e)) try: host = common.OpTestHost.OpTestHost(self.args.host_ip, self.args.host_user, self.args.host_password, self.args.bmc_ip, self.output, scratch_disk=self.args.host_scratch_disk, proxy=self.args.proxy, logfile=self.logfile, check_ssh_keys=self.args.check_ssh_keys, known_hosts_file=self.args.known_hosts_file, conf=self) if self.args.bmc_type in ['AMI', 'SMC']: web = OpTestWeb(self.args.bmc_ip, self.args.bmc_usernameipmi, self.args.bmc_passwordipmi) bmc = None if self.args.bmc_type in ['AMI']: ipmi = OpTestIPMI(self.args.bmc_ip, self.args.bmc_usernameipmi, self.args.bmc_passwordipmi, host=host, host_console_command=self.args.host_serial_console_command, logfile=self.logfile, ) bmc = OpTestBMC(ip=self.args.bmc_ip, username=self.args.bmc_username, password=self.args.bmc_password, logfile=self.logfile, ipmi=ipmi, web=web, check_ssh_keys=self.args.check_ssh_keys, known_hosts_file=self.args.known_hosts_file ) elif self.args.bmc_type in ['SMC']: ipmi = OpTestSMCIPMI(self.args.bmc_ip, self.args.bmc_usernameipmi, self.args.bmc_passwordipmi, logfile=self.logfile, host=host, ) bmc = OpTestSMC(ip=self.args.bmc_ip, username=self.args.bmc_username, password=self.args.bmc_password, ipmi=ipmi, web=web, check_ssh_keys=self.args.check_ssh_keys, known_hosts_file=self.args.known_hosts_file ) self.op_system = common.OpTestSystem.OpTestSystem( state=self.startState, bmc=bmc, host=host, conf=self, ) ipmi.set_system(self.op_system) bmc.set_system(self.op_system) elif self.args.bmc_type in ['FSP']: ipmi = OpTestIPMI(self.args.bmc_ip, None, # FSP does not use UID self.args.bmc_passwordipmi, host=host, logfile=self.logfile) bmc = OpTestFSP(self.args.bmc_ip, self.args.bmc_username, self.args.bmc_password, ipmi=ipmi, ) self.op_system = common.OpTestSystem.OpTestFSPSystem( state=self.startState, bmc=bmc, host=host, conf=self, ) ipmi.set_system(self.op_system) elif self.args.bmc_type in ['FSP_PHYP']: hmc = None if all(v is not None for v in [self.args.hmc_ip, self.args.hmc_username, self.args.hmc_password]): hmc = OpTestHMC(self.args.hmc_ip, self.args.hmc_username, self.args.hmc_password, managed_system=self.args.system_name, lpar_name=self.args.lpar_name, lpar_vios=self.args.lpar_vios, lpar_prof=self.args.lpar_prof, lpar_user=self.args.host_user, lpar_password=self.args.host_password, logfile=self.logfile ) else: raise Exception( "HMC IP, username and password is required") bmc = OpTestFSP(self.args.bmc_ip, self.args.bmc_username, self.args.bmc_password, ) self.op_system = common.OpTestSystem.OpTestFSPSystem( state=self.startState, bmc=bmc, host=host, hmc=hmc, conf=self, ) hmc.set_system(self.op_system) elif self.args.bmc_type in ['OpenBMC']: ipmi = OpTestIPMI(self.args.bmc_ip, self.args.bmc_usernameipmi, self.args.bmc_passwordipmi, host=host, logfile=self.logfile) rest_api = HostManagement(conf=self, ip=self.args.bmc_ip, username=self.args.bmc_username, password=self.args.bmc_password) bmc = OpTestOpenBMC(ip=self.args.bmc_ip, username=self.args.bmc_username, password=self.args.bmc_password, ipmi=ipmi, rest_api=rest_api, logfile=self.logfile, check_ssh_keys=self.args.check_ssh_keys, known_hosts_file=self.args.known_hosts_file) self.op_system = common.OpTestSystem.OpTestOpenBMCSystem( host=host, bmc=bmc, state=self.startState, conf=self, ) bmc.set_system(self.op_system) elif self.args.bmc_type in ['qemu']: print((repr(self.args))) bmc = OpTestQemu(conf=self, qemu_binary=self.args.qemu_binary, pnor=self.args.host_pnor, skiboot=self.args.flash_skiboot, kernel=self.args.flash_kernel, initramfs=self.args.flash_initramfs, cdrom=self.args.os_cdrom, logfile=self.logfile) self.op_system = common.OpTestSystem.OpTestQemuSystem(host=host, bmc=bmc, state=self.startState, conf=self, ) bmc.set_system(self.op_system) elif self.args.bmc_type in ['mambo']: if not (os.stat(self.args.mambo_binary).st_mode & stat.S_IXOTH): raise ParameterCheck(message="Check that the file exists with X permissions mambo-binary={}" .format(self.args.mambo_binary)) if self.args.flash_skiboot is None \ or not os.access(self.args.flash_skiboot, os.R_OK): raise ParameterCheck(message="Check that the file exists with R permissions flash-skiboot={}" .format(self.args.flash_skiboot)) if self.args.flash_kernel is None \ or not os.access(self.args.flash_kernel, os.R_OK): raise ParameterCheck(message="Check that the file exists with R permissions flash-kernel={}" .format(self.args.flash_kernel)) bmc = OpTestMambo(mambo_binary=self.args.mambo_binary, mambo_initial_run_script=self.args.mambo_initial_run_script, mambo_autorun=self.args.mambo_autorun, skiboot=self.args.flash_skiboot, kernel=self.args.flash_kernel, initramfs=self.args.flash_initramfs, timeout_factor=self.args.mambo_timeout_factor, logfile=self.logfile) self.op_system = common.OpTestSystem.OpTestMamboSystem(host=host, bmc=bmc, state=self.startState, conf=self, ) bmc.set_system(self.op_system) # Check that the bmc_type exists in our loaded addons then create our objects elif self.args.bmc_type in optAddons: (bmc, self.op_system) = optAddons[self.args.bmc_type].createSystem( self, host) else: self.util.cleanup() raise Exception("Unsupported BMC Type '{}', check your " "upper/lower cases for bmc_type and verify " "any credentials used from HostLocker or " "AES Version (see aes_get_creds " "version_mappings)".format(self.args.bmc_type)) host.set_system(self.op_system) return except Exception as e: traceback.print_exc() self.util.cleanup() raise e def bmc(self): return self.op_system.bmc def hmc(self): return self.op_system.hmc def system(self): return self.op_system def host(self): return self.op_system.host() def ipmi(self): return self.op_system.ipmi() def lspci_file(self): return self.args.host_lspci def platform(self): return self.args.platform
class OpTestIPMIReprovision(): ## Initialize this object # @param i_bmcIP The IP address of the BMC # @param i_bmcUser The userid to log into the BMC with # @param i_bmcPasswd The password of the userid to log into the BMC with # @param i_bmcUserIpmi The userid to issue the BMC IPMI commands with # @param i_bmcPasswdIpmi The password of BMC IPMI userid # @param i_ffdcDir Optional param to indicate where to write FFDC # # "Only required for inband tests" else Default = None # @param i_hostIP The IP address of the HOST # @param i_hostUser The userid to log into the HOST # @param i_hostPasswd The password of the userid to log into the HOST with # def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_hostIP=None, i_hostUser=None, i_hostPasswd=None): self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir) self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostIP, i_hostUser, i_hostPasswd) self.cv_HOST = OpTestHost(i_hostIP, i_hostUser, i_hostPasswd,i_bmcIP, i_ffdcDir) self.cv_SYSTEM = OpTestSystem(i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostIP, i_hostUser, i_hostPasswd) self.util = OpTestUtil() self.opTestHMIHandling = OpTestHMIHandling(i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostIP, i_hostUser, i_hostPasswd) ## # @brief This function will cover following test steps # Testcase: NVRAM Partition-IPMI Reprovision # 1. Update NVRAM config data with test config data # i.e "nvram --update-config test-name=test-value" # 2. Issue an IPMI PNOR Reprovision request command, to reset NVRAM partition to default. # 3. Wait for PNOR Reprovision progress to complete(00). # 4. Do a Hard reboot(Power OFF/ON) to avoid nvram cache data. # 5. Once system booted, check for NVRAM parition whether the test config data # got erased or not. # # @return BMC_CONST.FW_SUCCESS or raise OpTestError # def test_nvram_ipmi_reprovision(self): self.cv_SYSTEM.sys_bmc_power_on_validate_host() self.cv_HOST.host_run_command("uname -a") self.cv_HOST.host_run_command(BMC_CONST.NVRAM_PRINT_CFG) print "IPMI_Reprovision: Updating the nvram partition with test cfg data" self.cv_HOST.host_run_command(BMC_CONST.NVRAM_UPDATE_CONFIG_TEST_DATA) self.cv_HOST.host_run_command(BMC_CONST.NVRAM_PRINT_CFG) print "IPMI_Reprovision: issuing ipmi pnor reprovision request" self.cv_SYSTEM.sys_issue_ipmi_pnor_reprovision_request() print "IPMI_Reprovision: wait for reprovision to complete" self.cv_SYSTEM.sys_wait_for_ipmi_pnor_reprovision_to_complete() print "IPMI_Reprovision: gathering the opal message logs" self.cv_HOST.host_gather_opal_msg_log() print "IPMI_Reprovision: Performing a IPMI Power OFF Operation" # Perform a IPMI Power OFF Operation(Immediate Shutdown) self.cv_IPMI.ipmi_power_off() if int(self.cv_SYSTEM.sys_wait_for_standby_state(BMC_CONST.SYSTEM_STANDBY_STATE_DELAY)) == BMC_CONST.FW_SUCCESS: print "IPMI_Reprovision: System is in standby/Soft-off state" else: l_msg = "IPMI_Reprovision: System failed to reach standby/Soft-off state after pnor reprovisioning" raise OpTestError(l_msg) print "IPMI_Reprovision: Performing a IPMI Power ON Operation" # Perform a IPMI Power ON Operation self.cv_IPMI.ipmi_power_on() self.cv_SYSTEM.sys_check_host_status() self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE) l_res = self.cv_HOST.host_run_command(BMC_CONST.NVRAM_PRINT_CFG) if l_res.__contains__(BMC_CONST.NVRAM_TEST_DATA): l_msg = "NVRAM Partition - IPMI Reprovision not happening, nvram test config data still exists" raise OpTestError(l_msg) print "NVRAM Partition - IPMI Reprovision is done, cleared the nvram test config data" self.cv_HOST.host_gather_opal_msg_log() return BMC_CONST.FW_SUCCESS ## # @brief This function will cover following test steps # Testcase: GARD Partition-IPMI Reprovision # 1. Inject core checkstop using existed function from OpTestHMIHandling.py # 2. Do a Hard reboot(IPMI Power OFF/ON) # 2. Issue an IPMI PNOR Reprovision request command, to reset GUARD partition to default. # 3. Wait for IPMI PNOR Reprovision progress to complete(00). # 4. Check for GUARD parition whether the existing gard records erased or not. # 6. Reboot the system back to see system is booting fine or not. # # @return BMC_CONST.FW_SUCCESS or raise OpTestError # def test_gard_ipmi_reprovision(self): self.cv_SYSTEM.sys_bmc_power_on_validate_host() print "IPMI_Reprovision: Injecting system core checkstop to guard the phyisical cpu" self.opTestHMIHandling.testHMIHandling(BMC_CONST.HMI_MALFUNCTION_ALERT) print "IPMI_Reprovision: Performing a IPMI Power OFF Operation" # Perform a IPMI Power OFF Operation(Immediate Shutdown) self.cv_IPMI.ipmi_power_off() if int(self.cv_SYSTEM.sys_wait_for_standby_state(BMC_CONST.SYSTEM_STANDBY_STATE_DELAY)) == BMC_CONST.FW_SUCCESS: print "IPMI_Reprovision: System is in standby/Soft-off state" else: l_msg = "IPMI_Reprovision: System failed to reach standby/Soft-off state" raise OpTestError(l_msg) print "IPMI_Reprovision: Performing a IPMI Power ON Operation" # Perform a IPMI Power ON Operation self.cv_IPMI.ipmi_power_on() self.cv_SYSTEM.sys_check_host_status() self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE) print "IPMI_Reprovision: issuing ipmi pnor reprovision request" self.cv_SYSTEM.sys_issue_ipmi_pnor_reprovision_request() print "IPMI_Reprovision: wait for reprovision to complete" self.cv_SYSTEM.sys_wait_for_ipmi_pnor_reprovision_to_complete() print "IPMI_Reprovision: gathering the opal message logs" self.cv_HOST.host_gather_opal_msg_log() self.cv_HOST.host_get_OS_Level() self.cv_HOST.host_clone_skiboot_source(BMC_CONST.CLONE_SKIBOOT_DIR) # Compile the necessary tools xscom-utils and gard utility self.cv_HOST.host_compile_gard_utility(BMC_CONST.CLONE_SKIBOOT_DIR) l_con = self.cv_SYSTEM.sys_get_ipmi_console() self.cv_IPMI.ipmi_host_login(l_con) self.cv_IPMI.ipmi_host_set_unique_prompt(l_con) self.cv_IPMI.run_host_cmd_on_ipmi_console("uname -a") l_res = self.cv_SYSTEM.sys_list_gard_records(BMC_CONST.GARD_TOOL_DIR) print l_res if BMC_CONST.NO_GARD_RECORDS in l_res: print "GUARD Partition - IPMI Reprovision is done, cleared HW gard records" else: l_msg = "IPMI: Reprovision not happening, gard records are not erased" raise OpTestError(l_msg) print "IPMI_Reprovision: Performing a IPMI Power OFF Operation" # Perform a IPMI Power OFF Operation(Immediate Shutdown) self.cv_IPMI.ipmi_power_off() if int(self.cv_SYSTEM.sys_wait_for_standby_state(BMC_CONST.SYSTEM_STANDBY_STATE_DELAY)) == BMC_CONST.FW_SUCCESS: print "IPMI_Reprovision: System is in standby/Soft-off state" else: l_msg = "IPMI_Reprovision: System failed to reach standby/Soft-off state" raise OpTestError(l_msg) print "IPMI_Reprovision: Performing a IPMI Power ON Operation" # Perform a IPMI Power ON Operation self.cv_IPMI.ipmi_power_on() self.cv_SYSTEM.sys_check_host_status() self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE) print "IPMI_Reprovision: gathering the opal message logs" self.cv_HOST.host_gather_opal_msg_log() return BMC_CONST.FW_SUCCESS
def setUp(self): conf = OpTestConfiguration.conf self.cv_BMC = conf.bmc() self.cv_SYSTEM = conf.system() self.util = OpTestUtil() self.prompt = self.util.build_prompt()
class OpTestIPMIPowerControl(): ## Initialize this object # @param i_bmcIP The IP address of the BMC # @param i_bmcUser The userid to log into the BMC with # @param i_bmcPasswd The password of the userid to log into the BMC with # @param i_bmcUserIpmi The userid to issue the BMC IPMI commands with # @param i_bmcPasswdIpmi The password of BMC IPMI userid # @param i_ffdcDir Optional param to indicate where to write FFDC # # "Only required for inband tests" else Default = None # @param i_hostIP The IP address of the HOST # @param i_hostuser The userid to log into the HOST # @param i_hostPasswd The password of the userid to log into the HOST with # def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_hostip=None, i_hostuser=None, i_hostPasswd=None): self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir) self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostip, i_hostuser, i_hostPasswd) self.cv_HOST = OpTestHost(i_hostip, i_hostuser, i_hostPasswd, i_bmcIP) self.cv_SYSTEM = OpTestSystem(i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostip, i_hostuser, i_hostPasswd) self.util = OpTestUtil() ## # @brief This function will test below system power control operations # IPMI Power ON # Power OFF # Power Soft # Power Cycle # Power Reset # So each operation is executed through ipmi commands. and # check_system_status function will check whether FW and Host OS # Boot completed or not. # # @return BMC_CONST.FW_SUCCESS or raise OpTestError # def testIPMIPowerControl(self): self.cv_SYSTEM.sys_bmc_power_on_validate_host() print "Performing a IPMI Power OFF Operation" # Perform a IPMI Power OFF Operation(Immediate Shutdown) self.cv_IPMI.ipmi_power_off() if int(self.cv_SYSTEM.sys_wait_for_standby_state(BMC_CONST.SYSTEM_STANDBY_STATE_DELAY)) == 0: print "System is in standby/Soft-off state" else: l_msg = "System failed to reach standby/Soft-off state" raise OpTestError(l_msg) print "Performing a IPMI Power ON Operation" # Perform a IPMI Power ON Operation self.cv_IPMI.ipmi_power_on() self.check_system_status() self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE) print "Performing a IPMI Soft Power OFF Operation" # Perform a IPMI Soft Power OFF Operation(Graceful shutdown) self.cv_IPMI.ipmi_power_soft() if int(self.cv_SYSTEM.sys_wait_for_standby_state(BMC_CONST.SYSTEM_STANDBY_STATE_DELAY)) == 0: print "System is in standby/Soft-off state" else: l_msg = "System failed to reach standby/Soft-off state" raise OpTestError(l_msg) print "Perform a IPMI Power ON Operation" # Perform a IPMI Power ON Operation self.cv_IPMI.ipmi_power_on() self.check_system_status() self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE) print "Performing a IPMI Power Cycle(Soft reboot) Operation " # Perform a IPMI Power Cycle(Soft reboot) Operation only when system is in ON state self.cv_IPMI.ipmi_power_cycle() self.check_system_status() self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE) print "Performing a IPMI Power Hard Reset Operation" # Perform a IPMI Power Hard Reset Operation self.cv_IPMI.ipmi_power_reset() self.check_system_status() self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE) return BMC_CONST.FW_SUCCESS ## # @brief This function will check for system status and wait for # FW and Host OS Boot progress to complete. # # @return BMC_CONST.FW_SUCCESS or raise OpTestError # def check_system_status(self): if int(self.cv_SYSTEM.sys_ipl_wait_for_working_state()) == 0: print "System booted to working state" else: l_msg = "System failed to boot" raise OpTestError(l_msg) if int(self.cv_SYSTEM.sys_wait_for_os_boot_complete()) == 0: print "System booted to Host OS" else: l_msg = "System failed to boot Host OS" raise OpTestError(l_msg) return BMC_CONST.FW_SUCCESS
class HMCUtil(): ''' Utility and functions of HMC object ''' def __init__(self, hmc_ip, user_name, password, scratch_disk="", proxy="", logfile=sys.stdout, managed_system=None, lpar_name=None, prompt=None, block_setup_term=None, delaybeforesend=None, timeout_factor=None, lpar_prof=None, lpar_vios=None, lpar_user=None, lpar_password=None, check_ssh_keys=False, known_hosts_file=None, tgt_managed_system=None): self.hmc_ip = hmc_ip self.user = user_name self.passwd = password self.logfile = logfile self.mg_system = managed_system self.tgt_mg_system = tgt_managed_system self.check_ssh_keys = check_ssh_keys self.known_hosts_file = known_hosts_file self.lpar_name = lpar_name self.lpar_prof = lpar_prof self.lpar_user = lpar_user self.lpar_password = lpar_password self.lpar_vios = lpar_vios self.util = OpTestUtil() self.prompt = prompt self.expect_prompt = self.util.build_prompt(prompt) + "$" self.ssh = OpTestSSH(hmc_ip, user_name, password, logfile=self.logfile, check_ssh_keys=check_ssh_keys, known_hosts_file=known_hosts_file, block_setup_term=block_setup_term) self.scratch_disk = scratch_disk self.proxy = proxy self.scratch_disk_size = None self.delaybeforesend = delaybeforesend self.system = None # OpTestUtil instance is NOT conf's self.pty = None # allows caller specific control of when to block setup_term self.block_setup_term = block_setup_term # tells setup_term to not throw exceptions, like when system off self.setup_term_quiet = 0 # flags the object to abandon setup_term operations, like when system off self.setup_term_disable = 0 # functional simulators are very slow, so multiply all default timeouts by this factor self.timeout_factor = timeout_factor # state tracking, reset on boot and state changes # console tracking done on System object for the system console self.PS1_set = -1 self.LOGIN_set = -1 self.SUDO_set = -1 def deactivate_lpar_console(self): self.ssh.run_command("rmvterm -m %s -p %s" % (self.mg_system, self.lpar_name), timeout=10) def poweroff_system(self): if self.get_system_state() != OpManagedState.OPERATING: raise OpTestError('Managed Systen not in Operating state') self.ssh.run_command("chsysstate -m %s -r sys -o off" % self.mg_system) self.wait_system_state(OpManagedState.OFF) def poweron_system(self): if self.get_system_state() != OpManagedState.OFF: raise OpTestError('Managed Systen not is Power off state!') self.ssh.run_command("chsysstate -m %s -r sys -o on" % self.mg_system) self.wait_system_state() if self.lpar_vios: log.debug("Starting VIOS %s", self.lpar_vios) self.poweron_lpar(vios=True) def poweroff_lpar(self): if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]: log.info('LPAR Already powered-off!') return self.ssh.run_command( "chsysstate -m %s -r lpar -n %s -o shutdown --immed" % (self.mg_system, self.lpar_name)) self.wait_lpar_state(OpHmcState.NOT_ACTIVE) def poweron_lpar(self, vios=False): if self.get_lpar_state(vios) == OpHmcState.RUNNING: log.info('LPAR Already powered on!') return BMC_CONST.FW_SUCCESS lpar_name = self.lpar_name if vios: lpar_name = self.lpar_vios cmd = "chsysstate -m %s -r lpar -n %s -o on" % (self.mg_system, lpar_name) if self.lpar_prof: cmd = "%s -f %s" % (cmd, self.lpar_prof) self.wait_lpar_state(OpHmcState.NOT_ACTIVE, vios=vios) self.ssh.run_command(cmd) self.wait_lpar_state(vios=vios) time.sleep(STALLTIME) return BMC_CONST.FW_SUCCESS def dumprestart_lpar(self): if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]: log.info('LPAR Already powered-off!') return self.ssh.run_command("chsysstate -m %s -r lpar -n %s -o dumprestart" % (self.mg_system, self.lpar_name)) self.wait_lpar_state() def restart_lpar(self): if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]: log.info('LPAR Already powered-off!') return self.ssh.run_command( "chsysstate -m %s -r lpar -n %s -o shutdown --immed --restart" % (self.mg_system, self.lpar_name)) self.wait_lpar_state() def get_lpar_state(self, vios=False): lpar_name = self.lpar_name if vios: lpar_name = self.lpar_vios state = self.ssh.run_command( 'lssyscfg -m %s -r lpar --filter lpar_names=%s -F state' % (self.mg_system, lpar_name))[-1] ref_code = self.ssh.run_command( 'lsrefcode -m %s -r lpar --filter lpar_names=%s -F refcode' % (self.mg_system, lpar_name))[-1] if state == 'Running': if 'Linux' in ref_code or not ref_code: return 'Running' else: return 'Booting' return state def get_system_state(self): state = self.ssh.run_command('lssyscfg -m %s -r sys -F state' % self.mg_system) return state[-1] def wait_lpar_state(self, exp_state=OpHmcState.RUNNING, vios=False, timeout=WAITTIME): state = self.get_lpar_state(vios) count = 0 while state != exp_state: state = self.get_lpar_state(vios) log.info("Current state: %s", state) time.sleep(timeout) count = 1 if count > 120: raise OpTestError("Time exceeded for reaching %s" % exp_state) def wait_system_state(self, exp_state=OpManagedState.OPERATING, timeout=WAITTIME): state = self.get_system_state() count = 0 while state != exp_state: state = self.get_system_state() log.info("Current state: %s", state) time.sleep(timeout) count = 1 if count > 60: raise OpTestError("Time exceeded for reaching %s" % exp_state) def is_lpar_in_managed_system(self, mg_system=None): lpar_list = self.ssh.run_command('lssyscfg -r lpar -m %s -F name' % mg_system) if self.lpar_name in lpar_list: log.info("%s lpar found in managed system %s" % (mg_system, self.lpar_name)) return True return False def migrate_lpar(self, src_mg_system=None, dest_mg_system=None): if src_mg_system == None or dest_mg_system == None: raise OpTestError( "Source and Destination Managed System required for LPM") if not self.is_lpar_in_managed_system(src_mg_system): raise OpTestError("Lpar %s not found in managed system %s" % (self.lpar_name, src_mg_system)) self.ssh.run_command('migrlpar -o v -m %s -t %s -p %s' % (src_mg_system, dest_mg_system, self.lpar_name)) self.ssh.run_command('migrlpar -o m -m %s -t %s -p %s' % (src_mg_system, dest_mg_system, self.lpar_name)) if self.is_lpar_in_managed_system(dest_mg_system): log.info("Migration of lpar %s from %s to %s is successfull" % (self.lpar_name, src_mg_system, dest_mg_system)) self.mg_system = dest_mg_system return True log.info("Migration of lpar %s from %s to %s failed" % (self.lpar_name, src_mg_system, dest_mg_system)) return False def run_command_ignore_fail(self, command, timeout=60, retry=0): return self.ssh.run_command_ignore_fail(command, timeout * self.timeout_factor, retry) def run_command(self, i_cmd, timeout=15): return self.ssh.run_command(i_cmd, timeout)
class OpTestFWTS(): ## Initialize this object # @param i_bmcIP The IP address of the BMC # @param i_bmcUser The userid to log into the BMC with # @param i_bmcPasswd The password of the userid to log into the BMC with # @param i_bmcUserIpmi The userid to issue the BMC IPMI commands with # @param i_bmcPasswdIpmi The password of BMC IPMI userid # @param i_ffdcDir Optional param to indicate where to write FFDC # # "Only required for inband tests" else Default = None # @param i_hostIP The IP address of the HOST # @param i_hostuser The userid to log into the HOST # @param i_hostPasswd The password of the userid to log into the HOST with # def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_hostip=None, i_hostuser=None, i_hostPasswd=None): self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir) self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostip, i_hostuser, i_hostPasswd) self.cv_HOST = OpTestHost(i_hostip, i_hostuser, i_hostPasswd, i_bmcIP, i_ffdcDir) self.cv_SYSTEM = OpTestSystem(i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostip, i_hostuser, i_hostPasswd) self.util = OpTestUtil() self.user = i_hostuser self.ip = i_hostip self.passwd = i_hostPasswd ## # @brief This function just brings the system to host OS. # # @return BMC_CONST.FW_SUCCESS or raise OpTestError # def test_system_reboot(self): print "Testing FWTS: Booting system to OS" print "Performing a IPMI Power OFF Operation" # Perform a IPMI Power OFF Operation(Immediate Shutdown) self.cv_IPMI.ipmi_power_off() if int(self.cv_SYSTEM.sys_wait_for_standby_state(BMC_CONST.SYSTEM_STANDBY_STATE_DELAY)) == BMC_CONST.FW_SUCCESS: print "System is in standby/Soft-off state" else: l_msg = "System failed to reach standby/Soft-off state" raise OpTestError(l_msg) self.cv_IPMI.ipmi_power_on() self.cv_SYSTEM.sys_check_host_status() self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE) self.cv_IPMI.clear_ssh_keys(self.cv_HOST.ip) print "Gathering the OPAL msg logs" self.cv_HOST.host_gather_opal_msg_log() return BMC_CONST.FW_SUCCESS ## # @brief This function just executes the fwts_execution.sh on host OS # # @return BMC_CONST.FW_SUCCESS or raise OpTestError # def test_fwts(self): l_oslevel = self.cv_HOST.host_get_OS_Level() if not "Ubuntu" in l_oslevel: return # Copy the fwts execution file to the tmp folder in the host base_path = (os.path.dirname(os.path.abspath(__file__))).split('testcases')[0] fwts_script = base_path + "/testcases/fwts_execution.sh" try: self.util.copyFilesToDest(fwts_script, self.user, self.ip, "/tmp/", self.passwd) except: l_msg = "Copying fwts file to host failed" print l_msg raise OpTestError(l_msg) l_res = self.cv_HOST.host_run_command("/tmp/fwts_execution.sh") print l_res
class OpTestMtdPnorDriver(): ## Initialize this object and also getting the host login credentials to use by scp utility # @param i_bmcIP The IP address of the BMC # @param i_bmcUser The userid to log into the BMC with # @param i_bmcPasswd The password of the userid to log into the BMC with # @param i_bmcUserIpmi The userid to issue the BMC IPMI commands with # @param i_bmcPasswdIpmi The password of BMC IPMI userid # @param i_ffdcDir Optional param to indicate where to write FFDC # # "Only required for inband tests" else Default = None # @param i_hostIP The IP address of the HOST # @param i_hostuser The userid to log into the HOST # @param i_hostPasswd The password of the userid to log into the HOST with # def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_hostip=None, i_hostuser=None, i_hostPasswd=None): self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir) self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir) self.cv_HOST = OpTestHost(i_hostip, i_hostuser, i_hostPasswd, i_bmcIP) self.cv_SYSTEM = OpTestSystem(i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostip, i_hostuser, i_hostPasswd) self.util = OpTestUtil() self.host_user = i_hostuser self.host_ip = i_hostip self.host_Passwd = i_hostPasswd ## # @brief This function has following test steps # 1. Get host information(OS and Kernel information) # 2. Load the mtd module based on config value # 3. Check /dev/mtd0 character device file existence on host # 4. Copying the contents of the flash in a file /tmp/pnor # 5. Getting the /tmp/pnor file into local x86 machine using scp utility # 6. Remove existing /tmp/ffs directory and # Clone latest ffs git repository in local x86 working machine # 7. Compile ffs repository to get fcp utility # 8. Check existence of fcp utility in ffs repository, after compiling # 9. Get the PNOR flash contents on an x86 machine using fcp utility # # @return BMC_CONST.FW_SUCCESS-success or raise OpTestError-fail # def testMtdPnorDriver(self): self.cv_SYSTEM.sys_bmc_power_on_validate_host() # Get OS level l_oslevel = self.cv_HOST.host_get_OS_Level() # Get Kernel Version l_kernel = self.cv_HOST.host_get_kernel_version() # loading mtd module based on config option l_config = "CONFIG_MTD_POWERNV_FLASH" l_module = "mtd" self.cv_HOST.host_load_module_based_on_config(l_kernel, l_config, l_module) # Check /dev/mtd0 file existence on host l_cmd = "ls -l /dev/mtd0; echo $?" l_res = self.cv_HOST.host_run_command(l_cmd) l_res = l_res.splitlines() if int(l_res[-1]) == 0: print "/dev/mtd0 character device file exists on host" else: l_msg = "/dev/mtd0 character device file doesn't exist on host" print l_msg raise OpTestError(l_msg) # Copying the contents of the PNOR flash in a file /tmp/pnor l_file = "/tmp/pnor" l_cmd = "cat /dev/mtd0 > %s; echo $?" % l_file l_res = self.cv_HOST.host_run_command(l_cmd) l_res = l_res.splitlines() if int(l_res[-1]) == 0: print "Fetched PNOR data from /dev/mtd0 into temp file /tmp/pnor" else: l_msg = "Fetching PNOR data is failed from /dev/mtd0 into temp file /tmp/pnor" print l_msg raise OpTestError(l_msg) # Getting the /tmp/pnor file into local x86 machine l_path = "/tmp/" self.util.copyFilesToDest(l_path, self.host_user, self.host_ip, l_file, self.host_Passwd, "2", BMC_CONST.SCP_TO_LOCAL) l_list = commands.getstatusoutput("ls -l %s; echo $?" % l_path) print l_list l_workdir = "/tmp/ffs" # Remove existing /tmp/ffs directory l_res = commands.getstatusoutput("rm -rf %s" % l_workdir) print l_res # Clone latest ffs git repository in local x86 working machine l_cmd = "git clone https://github.com/open-power/ffs/ %s" % l_workdir l_res = commands.getstatusoutput(l_cmd) print l_res if int(l_res[0]) == 0: print "Cloning of ffs repository is successfull" else: l_msg = "Cloning ffs repository is failed" print l_msg raise OpTestError(l_msg) # Compile ffs repository to get fcp utility l_cmd = "cd %s/; make" % l_workdir l_res = commands.getstatusoutput(l_cmd) print l_res if int(l_res[0]) == 0: print "Compiling fcp utility is successfull" else: l_msg = "Compiling fcp utility is failed" print l_msg raise OpTestError(l_msg) # Check existence of fcp utility in ffs repository, after compiling. l_cmd = "test -f %s/fcp/x86/fcp" % l_workdir l_res = commands.getstatusoutput(l_cmd) print l_res if int(l_res[0]) == 0: print "Compiling fcp utility is successfull" else: l_msg = "Compiling fcp utility is failed" print l_msg raise OpTestError(l_msg) # Check the PNOR flash contents on an x86 machine using fcp utility l_cmd = "%s/fcp/x86/fcp -o 0x0 -L %s" % (l_workdir, l_file) l_res = commands.getstatusoutput(l_cmd) print l_res[1] if int(l_res[0]) == 0: print "Getting PNOR data successfull using fcp utility" return BMC_CONST.FW_SUCCESS else: l_msg = "Getting the PNOR data using fcp utility failed" print l_msg raise OpTestError(l_msg)
class HMCConsole(HMCUtil): """ HMCConsole Class Methods to manage the console of LPAR """ def __init__(self, hmc_ip, user_name, password, managed_system, lpar_name, lpar_vios, lpar_prof, lpar_user, lpar_password, block_setup_term=None, delaybeforesend=None, timeout_factor=1, logfile=sys.stdout, prompt=None, scratch_disk="", check_ssh_keys=False, known_hosts_file=None, proxy=""): self.logfile = logfile self.hmc_ip = hmc_ip self.user = user_name self.passwd = password self.mg_system = managed_system self.util = OpTestUtil() self.expect_prompt = self.util.build_prompt(prompt) + "$" self.lpar_name = lpar_name self.lpar_vios = lpar_vios self.lpar_prof = lpar_prof self.lpar_user = lpar_user self.lpar_password = lpar_password self.scratch_disk = scratch_disk self.proxy = proxy self.state = ConsoleState.DISCONNECTED self.delaybeforesend = delaybeforesend self.system = None # OpTestUtil instance is NOT conf's self.prompt = prompt self.pty = None self.delaybeforesend = delaybeforesend # allows caller specific control of when to block setup_term self.block_setup_term = block_setup_term # tells setup_term to not throw exceptions, like when system off self.setup_term_quiet = 0 # flags the object to abandon setup_term operations, like when system off self.setup_term_disable = 0 # FUTURE - System Console currently tracked in System Object # state tracking, reset on boot and state changes self.PS1_set = -1 self.LOGIN_set = -1 self.SUDO_set = -1 super(HMCConsole, self).__init__(hmc_ip, user_name, password, scratch_disk, proxy, logfile, managed_system, lpar_name, prompt, block_setup_term, delaybeforesend, timeout_factor, lpar_prof, lpar_vios, lpar_user, lpar_password, check_ssh_keys, known_hosts_file) def set_system(self, system): self.ssh.set_system(system) self.system = system self.pty = self.get_console() self.pty.set_system(system) def get_host_console(self): return self.pty def set_system_setup_term(self, flag): self.system.block_setup_term = flag def get_system_setup_term(self): return self.system.block_setup_term def get_scratch_disk(self): return self.scratch_disk def get_proxy(self): return self.proxy def hostname(self): return self.hmc_ip def username(self): return self.user def password(self): return self.passwd def set_block_setup_term(self, flag): self.block_setup_term = flag def get_block_setup_term(self): return self.block_setup_term def enable_setup_term_quiet(self): self.setup_term_quiet = 1 self.setup_term_disable = 0 def disable_setup_term_quiet(self): self.setup_term_quiet = 0 self.setup_term_disable = 0 def close(self): self.util.clear_state(self) try: self.pty.close() if self.pty.status != -1: # leaving for debug if os.WIFEXITED(self.pty.status): os.WEXITSTATUS(self.pty.status) else: os.WTERMSIG(self.pty.status) self.state = ConsoleState.DISCONNECTED except pexpect.ExceptionPexpect: self.state = ConsoleState.DISCONNECTED raise "HMC Console: failed to close console" except Exception: self.state = ConsoleState.DISCONNECTED log.debug("HMC close -> TERMINATE") def connect(self, logger=None): if self.state == ConsoleState.CONNECTED: return self.pty self.util.clear_state(self) # clear when coming in DISCONNECTED log.info("De-activating the console") self.deactivate_lpar_console() log.debug("#HMC Console CONNECT") command = "sshpass -p %s ssh -p 22 -l %s %s -o PubkeyAuthentication=no"\ " -o afstokenpassing=no -q -o 'UserKnownHostsFile=/dev/null'"\ " -o 'StrictHostKeyChecking=no'" try: self.pty = Spawn(command % (self.passwd, self.user, self.hmc_ip)) log.info("Opening the LPAR console") time.sleep(STALLTIME) self.pty.send('\r') self.pty.sendline("mkvterm -m %s -p %s" % (self.mg_system, self.lpar_name)) self.pty.send('\r') time.sleep(STALLTIME) i = self.pty.expect(["Open Completed.", pexpect.TIMEOUT], timeout=60) self.pty.logfile = sys.stdout if logger: self.pty.logfile_read = OpTestLogger.FileLikeLogger(logger) else: self.pty.logfile_read = OpTestLogger.FileLikeLogger(log) if i == 0: time.sleep(STALLTIME) self.state = ConsoleState.CONNECTED self.pty.setwinsize(1000, 1000) else: raise OpTestError("Check the lpar activate command") except Exception as exp: self.state = ConsoleState.DISCONNECTED raise CommandFailed( 'OPexpect.spawn', 'OPexpect.spawn encountered a problem: ' + str(exp), -1) if self.delaybeforesend: self.pty.delaybeforesend = self.delaybeforesend if not self.pty.isalive(): raise CommandFailed("mkvterm", self.pty.read(), self.pty.status) return self.pty def check_state(self): return self.state def get_console(self, logger=None): if self.state == ConsoleState.DISCONNECTED: self.util.clear_state(self) self.connect(logger=logger) time.sleep(STALLTIME) l_rc = self.pty.expect(["login:"******"Shell prompt changed") else: self.pty.send('\r') log.debug("Waiting till booting!") self.pty = self.get_login_prompt() if self.system.SUDO_set != 1 or self.system.LOGIN_set != 1 or self.system.PS1_set != 1: self.util.setup_term(self.system, self.pty, None, self.system.block_setup_term) # Clear buffer before usage self.pty.buffer = "" return self.pty def get_login_prompt(self): # Assuming 'Normal' boot set in LPAR profile # We wait for upto 500 seconds for LPAR to boot to OS self.pty.send('\r') time.sleep(STALLTIME) log.debug("Waiting for login screen") i = self.pty.expect(["login:"******"System has booted") time.sleep(STALLTIME) elif i == 1: log.debug("Console already logged in") else: log.error("Failed to get login prompt %s", self.pty.before) # To cheat system for making using of HMC SSH self.system.PS1_set = 1 self.system.LOGIN_set = 1 self.system.SUDO_set = 1 return self.pty def run_command(self, i_cmd, timeout=15): return self.util.run_command(self, i_cmd, timeout)
class OpTestFlashBase(unittest.TestCase): def setUp(self): conf = OpTestConfiguration.conf self.cv_SYSTEM = conf.system() self.cv_BMC = conf.bmc() self.cv_REST = self.cv_BMC.get_rest_api() self.cv_HOST = conf.host() self.cv_IPMI = conf.ipmi() self.platform = conf.platform() self.util = OpTestUtil() self.bmc_type = conf.args.bmc_type self.bmc_ip = conf.args.bmc_ip self.bmc_username = conf.args.bmc_username self.bmc_password = conf.args.bmc_password self.pupdate_binary = conf.args.pupdate self.pflash = conf.args.pflash def validate_side_activated(self): l_bmc_side, l_pnor_side = self.cv_IPMI.ipmi_get_side_activated() self.assertIn(BMC_CONST.PRIMARY_SIDE, l_bmc_side, "BMC: Primary side is not active") if (l_pnor_side == BMC_CONST.GOLDEN_SIDE): print "PNOR: Primary side is not active" bios_sensor = self.cv_IPMI.ipmi_get_golden_side_sensor_id() self.assertNotEqual( bios_sensor, None, "Failed to get the BIOS Golden side sensor id") boot_count_sensor = self.cv_IPMI.ipmi_get_boot_count_sensor_id() self.assertNotEqual(boot_count_sensor, None, "Failed to get the Boot Count sensor id") self.cv_IPMI.ipmi_set_pnor_primary_side(bios_sensor, boot_count_sensor) l_bmc_side, l_pnor_side = self.cv_IPMI.ipmi_get_side_activated() self.assertIn(BMC_CONST.PRIMARY_SIDE, l_pnor_side, "PNOR: Primary side is not active") def get_pnor_level(self): rc = self.cv_IPMI.ipmi_get_PNOR_level() print rc def bmc_down_check(self): self.assertTrue(self.util.ping_fail_check(self.cv_BMC.host_name), "FSP/BMC keeps on pinging up") def scp_file(self, src_file_path, dst_file_path): self.util.copyFilesToDest(src_file_path, self.bmc_username, self.bmc_ip, dst_file_path, self.bmc_password, "2", BMC_CONST.SCP_TO_REMOTE) def get_version_tar(self, file_path): tar = tarfile.open(file_path) for member in tar.getmembers(): fd = tar.extractfile(member) content = fd.read() if "version=" in content: content = content.split("\n") content = [x for x in content if "version=" in x] version = content[0].split("=")[-1] break tar.close() print version return version def get_image_version(self, path): output = self.cv_BMC.run_command("cat %s | grep \"version=\"" % path) return output[0].split("=")[-1] def delete_images_dir(self): try: self.cv_BMC.run_command("rm -rf /tmp/images/*") except CommandFailed: pass def get_image_path(self, image_version): retry = 0 while (retry < 20): image_list = [] try: image_list = self.cv_BMC.run_command( "ls -1 -d /tmp/images/*/ --color=never") except CommandFailed as cf: pass for i in range(0, len(image_list)): version = self.get_image_version(image_list[i] + "MANIFEST") if (version == image_version): return image_list[i] time.sleep(5) retry += 1 def get_image_id(self, version): img_path = self.get_image_path(version) img_id = img_path.split("/")[-2] print "Image id for Host image is : %s" % img_id return img_id def wait_for_bmc_runtime(self): self.util.PingFunc(self.bmc_ip, BMC_CONST.PING_RETRY_FOR_STABILITY) if "SMC" in self.bmc_type: self.cv_IPMI.ipmi_wait_for_bmc_runtime() elif "OpenBMC" in self.bmc_type: self.cv_REST.wait_for_bmc_runtime() return
class OpTestHMIHandling(unittest.TestCase): def setUp(self): conf = OpTestConfiguration.conf self.cv_HOST = conf.host() self.cv_IPMI = conf.ipmi() self.cv_FSP = conf.bmc() self.cv_SYSTEM = conf.system() self.bmc_type = conf.args.bmc_type self.util = OpTestUtil() def ipmi_monitor_sol_ipl(self, console, timeout): # Error injection causing the SOL console to terminate immediately. # So Let's re-connect the console console.close() self.cv_SYSTEM.set_state(OpSystemState.IPLing) try: self.cv_SYSTEM.goto_state(OpSystemState.OS) except: self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE) self.cv_SYSTEM.set_state(OpSystemState.OS) console.close() print "System booted fine to host OS..." return BMC_CONST.FW_SUCCESS def verify_proc_recovery(self, l_res): if any("Processor Recovery done" in line for line in l_res) and \ any("Harmless Hypervisor Maintenance interrupt [Recovered]" in line for line in l_res): print "Processor recovery done" return else: raise Exception( "HMI handling failed to log message: for proc_recv_done") def verify_timer_facility_recovery(self, l_res): if any("Timer facility experienced an error" in line for line in l_res) and \ any("Severe Hypervisor Maintenance interrupt [Recovered]" in line for line in l_res): print "Timer facility experienced an error and got recovered" return else: raise Exception("HMI handling failed to log message") def init_test(self): self.cv_SYSTEM.goto_state(OpSystemState.OS) self.proc_gen = self.cv_HOST.host_get_proc_gen() l_chips = self.cv_HOST.host_get_list_of_chips( ) # ['00000000', '00000001', '00000010'] if not l_chips: raise Exception("Getscom failed to list processor chip ids") l_cores = self.cv_HOST.host_get_cores() if not l_cores: raise Exception("Failed to get list of core id's") print l_cores # {0: ['4', '5', '6', 'c', 'd', 'e'], 1: ['4', '5', '6', 'c', 'd', 'e'], 10: ['4', '5', '6', 'c', 'd', 'e']} # Remove master core where injecting core checkstop leads to IPL expected failures # after 2 failures system will starts boot in Golden side of PNOR l_cores[0][1].pop(0) print l_cores self.l_dic = [] i = 0 for tup in l_cores: new_list = [l_chips[i], tup[1]] self.l_dic.append(new_list) i += 1 print self.l_dic # self.l_dic is a list of chip id's, core id's . and is of below format # [['00000000', ['4', '5', '6', 'c', 'd', 'e']], ['00000001', ['4', '5', '6', 'c', 'd', 'e']], ['00000010', ['4', '5', '6', 'c', 'd', 'e']]] # In-order to inject HMI errors on cpu's, cpu should be running, so disabling the sleep states 1 and 2 of all CPU's self.disable_cpu_idle_states() # Disable kdump to check behaviour of IPL caused due to kernel panic after injection of core/system checkstop self.disable_kdump_service() def disable_kdump_service(self): l_oslevel = self.cv_HOST.host_get_OS_Level() try: if "Ubuntu" in l_oslevel: self.cv_HOST.host_run_command("service kdump-tools stop") else: self.cv_HOST.host_run_command("service kdump stop") except CommandFailed as cf: if cf.exitcode == 5: # kdump may not be enabled, so it's not a failure to stop it pass def enable_idle_state(self, i_idle): l_cmd = "for i in /sys/devices/system/cpu/cpu*/cpuidle/state%s/disable; do echo 0 > $i; done" % i_idle self.cv_HOST.host_run_command(l_cmd) def disable_idle_state(self, i_idle): l_cmd = "for i in /sys/devices/system/cpu/cpu*/cpuidle/state%s/disable; do echo 1 > $i; done" % i_idle self.cv_HOST.host_run_command(l_cmd) # Disable all CPU idle states except snooze state def disable_cpu_idle_states(self): states = self.cv_HOST.host_run_command( "find /sys/devices/system/cpu/cpu*/cpuidle/state* -type d | cut -d'/' -f8 | sort -u | sed -e 's/^state//'" ) for state in states: if state is "0": try: self.cv_HOST.host_run_command("cpupower idle-set -e 0") except CommandFailed: self.enable_idle_state("0") continue try: self.cv_HOST.host_run_command("cpupower idle-set -d %s" % state) except CommandFailed: self.disable_idle_state(state) def form_scom_addr(self, addr, core): if self.proc_gen in ["POWER8", "POWER8E"]: val = addr[0] + str(core) + addr[2:] elif self.proc_gen in ["POWER9"]: val = hex( eval("0x%s | (((%s & 0x1f) + 0x20) << 24)" % (addr, int(core, 16)))) print val return val def clearGardEntries(self): self.cv_SYSTEM.goto_state(OpSystemState.OS) self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE) if "FSP" in self.bmc_type: res = self.cv_FSP.fspc.run_command("gard --clr all") self.assertIn("Success in clearing Gard Data", res, "Failed to clear GARD entries") print self.cv_FSP.fspc.run_command("gard --gc cpu") else: g = self.cv_HOST.host_run_command( "PATH=/usr/local/sbin:$PATH opal-gard list all") if "No GARD entries to display" not in g: self.cv_HOST.host_run_command( "PATH=/usr/local/sbin:$PATH opal-gard clear all") cleared_gard = self.cv_HOST.host_run_command( "PATH=/usr/local/sbin:$PATH opal-gard list") self.assertIn("No GARD entries to display", cleared_gard, "Failed to clear GARD entries") self.cv_SYSTEM.goto_state(OpSystemState.OFF) self.cv_SYSTEM.goto_state(OpSystemState.OS) ## # @brief This function executes HMI test case based on the i_test value, Before test starts # disabling kdump service to make sure system reboots, after injecting non-recoverable errors. # # @param i_test @type int: this is the type of test case want to execute # BMC_CONST.HMI_PROC_RECV_DONE: Processor recovery done # BMC_CONST.HMI_PROC_RECV_ERROR_MASKED: proc_recv_error_masked # BMC_CONST.HMI_MALFUNCTION_ALERT: malfunction_alert # BMC_CONST.HMI_HYPERVISOR_RESOURCE_ERROR: hypervisor resource error def _testHMIHandling(self, i_test): l_test = i_test self.init_test() self.cv_SYSTEM.goto_state(OpSystemState.OS) self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE) l_con = self.cv_SYSTEM.sys_get_ipmi_console() self.cv_SYSTEM.host_console_login() self.cv_SYSTEM.host_console_unique_prompt() l_con.run_command("uname -a") l_con.run_command("cat /etc/os-release") l_con.run_command("lscpu") l_con.run_command("dmesg -D") if l_test == BMC_CONST.HMI_PROC_RECV_DONE: self._test_proc_recv_done() elif l_test == BMC_CONST.HMI_PROC_RECV_ERROR_MASKED: self._test_proc_recv_error_masked() elif l_test == BMC_CONST.HMI_MALFUNCTION_ALERT: self._test_malfunction_allert() elif l_test == BMC_CONST.HMI_HYPERVISOR_RESOURCE_ERROR: self._test_hyp_resource_err() elif l_test == BMC_CONST.TOD_ERRORS: # TOD Error recovery works on systems having more than one chip TOD # Skip this test on single chip systems(as recovery fails on 1S systems) if len(self.l_dic) == 1: l_msg = "This is a single chip system, TOD Error recovery won't work" print l_msg return BMC_CONST.FW_SUCCESS elif len(self.l_dic) > 1: self._test_tod_errors(BMC_CONST.PSS_HAMMING_DISTANCE) self._test_tod_errors(BMC_CONST.INTERNAL_PATH_OR_PARITY_ERROR) self._test_tod_errors(BMC_CONST.TOD_DATA_PARITY_ERROR) self._test_tod_errors(BMC_CONST.TOD_SYNC_CHECK_ERROR) self._test_tod_errors(BMC_CONST.FSM_STATE_PARITY_ERROR) self._test_tod_errors(BMC_CONST.MASTER_PATH_CONTROL_REGISTER) self._test_tod_errors( BMC_CONST.PORT_0_PRIMARY_CONFIGURATION_REGISTER) self._test_tod_errors( BMC_CONST.PORT_1_PRIMARY_CONFIGURATION_REGISTER) self._test_tod_errors( BMC_CONST.PORT_0_SECONDARY_CONFIGURATION_REGISTER) self._test_tod_errors( BMC_CONST.PORT_1_SECONDARY_CONFIGURATION_REGISTER) self._test_tod_errors(BMC_CONST.SLAVE_PATH_CONTROL_REGISTER) self._test_tod_errors(BMC_CONST.INTERNAL_PATH_CONTROL_REGISTER) self._test_tod_errors(BMC_CONST.PR_SC_MS_SL_CONTROL_REGISTER) else: raise Exception("Getting Chip information failed") elif l_test == BMC_CONST.TFMR_ERRORS: self._testTFMR_Errors(BMC_CONST.TB_PARITY_ERROR) self._testTFMR_Errors(BMC_CONST.TFMR_PARITY_ERROR) self._testTFMR_Errors(BMC_CONST.TFMR_HDEC_PARITY_ERROR) self._testTFMR_Errors(BMC_CONST.TFMR_DEC_PARITY_ERROR) self._testTFMR_Errors(BMC_CONST.TFMR_PURR_PARITY_ERROR) self._testTFMR_Errors(BMC_CONST.TFMR_SPURR_PARITY_ERROR) else: raise Exception("Please provide valid test case") self.cv_HOST.ssh.state = SSHConnectionState.DISCONNECTED return BMC_CONST.FW_SUCCESS ## # @brief This function is used to test HMI: processor recovery done # and also this function injecting error on all the cpus one by one and # verify whether cpu is recovered or not. def _test_proc_recv_done(self): if self.proc_gen in ["POWER9"]: scom_addr = "20010A40" elif self.proc_gen in ["POWER8", "POWER8E"]: scom_addr = "10013100" else: return for l_pair in self.l_dic: l_chip = l_pair[0] for l_core in l_pair[1]: l_reg = self.form_scom_addr(scom_addr, l_core) l_cmd = "PATH=/usr/local/sbin:$PATH putscom -c %s %s 0000000000100000" % ( l_chip, l_reg) console = self.cv_SYSTEM.sys_get_ipmi_console() console.run_command("dmesg -C") try: l_res = console.run_command(l_cmd, timeout=20) except CommandFailed as cf: l_res = cf.output if cf.exitcode == 1: pass else: if any("Kernel panic - not syncing" in line for line in l_res): raise Exception( "Processor recovery failed: Kernel got panic") elif any("Petitboot" in line for line in l_res): raise Exception( "System reached petitboot:Processor recovery failed" ) elif any("ISTEP" in line for line in l_res): raise Exception( "System started booting: Processor recovery failed" ) else: raise Exception( "Failed to inject thread hang recoverable error %s", str(cf)) time.sleep(0.2) l_res = console.run_command("dmesg") self.verify_proc_recovery(l_res) return ## # @brief This function is used to test HMI: proc_recv_error_masked # Processor went through recovery for an error which is actually masked for reporting # this function also injecting the error on all the cpu's one-by-one. def _test_proc_recv_error_masked(self): if self.proc_gen in ["POWER8", "POWER8E"]: scom_addr = "10013100" else: return for l_pair in self.l_dic: l_chip = l_pair[0] for l_core in l_pair[1]: l_reg = self.form_scom_addr(scom_addr, l_core) l_cmd = "PATH=/usr/local/sbin:$PATH putscom -c %s %s 0000000000080000" % ( l_chip, l_reg) console = self.cv_SYSTEM.sys_get_ipmi_console() console.run_command("dmesg -C") try: l_res = console.run_command(l_cmd, timeout=20) except CommandFailed as cf: l_res = cf.output if cf.exitcode == 1: pass else: if any("Kernel panic - not syncing" in line for line in l_res): raise Exception( "Processor recovery failed: Kernel got panic") elif any("Petitboot" in line for line in l_res): raise Exception( "System reached petitboot:Processor recovery failed" ) elif any("ISTEP" in line for line in l_res): raise Exception( "System started booting: Processor recovery failed" ) else: raise Exception( "Failed to inject thread hang recoverable error %s", str(cf)) time.sleep(0.2) l_res = console.run_command("dmesg") self.verify_proc_recovery(l_res) return ## # @brief This function is used to test hmi malfunction alert:Core checkstop # A processor core in the system has to be checkstopped (failed recovery). # Injecting core checkstop on random core of random chip def _test_malfunction_allert(self): if self.proc_gen in ["POWER9"]: scom_addr = "20010A40" elif self.proc_gen in ["POWER8", "POWER8E"]: scom_addr = "10013100" else: return # Get random pair of chip vs cores l_pair = random.choice(self.l_dic) # Get random chip id l_chip = l_pair[0] # Get random core number l_core = random.choice(l_pair[1]) l_reg = self.form_scom_addr(scom_addr, l_core) l_cmd = "PATH=/usr/local/sbin:$PATH putscom -c %s %s 1000000000000000" % ( l_chip, l_reg) # Core checkstop will lead to system IPL, so we will wait for certain time for IPL # to finish #l_res = self.cv_SYSTEM.sys_get_ipmi_console().run_command(l_cmd, timeout=600) console = self.cv_SYSTEM.sys_get_ipmi_console() console.sol.sendline(l_cmd) self.ipmi_monitor_sol_ipl(console, timeout=600) ## # @brief This function is used to test HMI: Hypervisor resource error # Injecting Hypervisor resource error on random core of random chip def _test_hyp_resource_err(self): if self.proc_gen in ["POWER9"]: scom_addr = "20010A40" elif self.proc_gen in ["POWER8", "POWER8E"]: scom_addr = "10013100" else: return # Get random pair of chip vs cores l_pair = random.choice(self.l_dic) # Get random chip id l_chip = l_pair[0] # Get random core number l_core = random.choice(l_pair[1]) l_reg = self.form_scom_addr(scom_addr, l_core) l_cmd = "PATH=/usr/local/sbin:$PATH putscom -c %s %s 0000000000008000" % ( l_chip, l_reg) console = self.cv_SYSTEM.sys_get_ipmi_console() console.sol.sendline(l_cmd) self.ipmi_monitor_sol_ipl(console, timeout=600) ## # @brief This function tests timer facility related error injections and check # the corresponding error got recovered. And this process is repeated # for all the active cores in all the chips. # # @param i_error @type string: this is the type of error want to inject # BMC_CONST.TB_PARITY_ERROR # BMC_CONST.TFMR_PARITY_ERROR # BMC_CONST.TFMR_HDEC_PARITY_ERROR # BMC_CONST.TFMR_DEC_PARITY_ERROR # BMC_CONST.TFMR_PURR_PARITY_ERROR # BMC_CONST.TFMR_SPURR_PARITY_ERROR def _testTFMR_Errors(self, i_error): if self.proc_gen in ["POWER9"]: scom_addr = "20010A84" elif self.proc_gen in ["POWER8", "POWER8E"]: scom_addr = "10013281" else: return l_error = i_error for l_pair in self.l_dic: l_chip = l_pair[0] for l_core in l_pair[1]: l_reg = self.form_scom_addr(scom_addr, l_core) l_cmd = "PATH=/usr/local/sbin:$PATH putscom -c %s %s %s" % ( l_chip, l_reg, l_error) console = self.cv_SYSTEM.sys_get_ipmi_console() console.run_command("dmesg -C") try: l_res = console.run_command(l_cmd, timeout=20) except CommandFailed as cf: l_res = cf.output if cf.exitcode == 1: pass else: if any("Kernel panic - not syncing" in line for line in l_res): l_msg = "TFMR error injection: Kernel got panic" elif any("Petitboot" in line for line in l_res): l_msg = "System reached petitboot:TFMR error injection recovery failed" elif any("ISTEP" in line for line in l_res): l_msg = "System started booting: TFMR error injection recovery failed" else: raise Exception("Failed to inject TFMR error %s " % str(cf)) time.sleep(0.2) l_res = console.run_command("dmesg") self.verify_timer_facility_recovery(l_res) return ## # @brief This function tests chip TOD related error injections and check # the corresponding error got recovered. And this error injection # happening on a random chip. This tod errors should test on systems # having more than one processor socket(chip). On single chip system # TOD error recovery won't work. # # @param i_error @type string: this is the type of error want to inject # These errors represented in common/OpTestConstants.py file. def _test_tod_errors(self, i_error): l_error = i_error l_pair = random.choice(self.l_dic) # Get random chip id l_chip = l_pair[0] l_cmd = "PATH=/usr/local/sbin:$PATH putscom -c %s %s %s" % ( l_chip, BMC_CONST.TOD_ERROR_REG, l_error) console = self.cv_SYSTEM.sys_get_ipmi_console() console.run_command("dmesg -C") # As of now putscom command to TOD register will fail with return code -1. # putscom indirectly call getscom to read the value again. # But getscom to TOD error reg there is no access # TOD Error reg has only WO access and there is no read access try: l_res = console.run_command(l_cmd, timeout=20) except CommandFailed as cf: l_res = cf.output if cf.exitcode == 1: pass else: if any("Kernel panic - not syncing" in line for line in l_res): print "TOD ERROR Injection-kernel got panic" elif any("login:"******"System booted to host OS without any kernel panic message" elif any("Petitboot" in line for line in l_res): print "System reached petitboot without any kernel panic message" elif any("ISTEP" in line for line in l_res): print "System started booting without any kernel panic message" else: raise Exception( "TOD: PSS Hamming distance error injection failed %s", str(cf)) time.sleep(0.2) l_res = console.run_command("dmesg") self.verify_timer_facility_recovery(l_res) return ## # @brief This function enables a single core def host_enable_single_core(self): self.cv_HOST.host_enable_single_core()
class OpTestFlashBase(unittest.TestCase): def setUp(self): conf = OpTestConfiguration.conf self.cv_SYSTEM = conf.system() self.cv_BMC = conf.bmc() self.cv_REST = self.cv_BMC.get_rest_api() self.cv_HOST = conf.host() self.cv_IPMI = conf.ipmi() self.platform = conf.platform() self.util = OpTestUtil() self.bmc_type = conf.args.bmc_type self.bmc_ip = conf.args.bmc_ip self.bmc_username = conf.args.bmc_username self.bmc_password = conf.args.bmc_password self.pupdate_binary = conf.args.pupdate self.pflash = conf.args.pflash def validate_side_activated(self): l_bmc_side, l_pnor_side = self.cv_IPMI.ipmi_get_side_activated() self.assertIn(BMC_CONST.PRIMARY_SIDE, l_bmc_side, "BMC: Primary side is not active") # TODO force setting of primary side to BIOS Golden side sensor self.assertIn(BMC_CONST.PRIMARY_SIDE, l_pnor_side, "PNOR: Primary side is not active") def get_pnor_level(self): rc = self.cv_IPMI.ipmi_get_PNOR_level() print rc def bmc_down_check(self): self.assertTrue(self.util.ping_fail_check(self.cv_BMC.host_name), "FSP/BMC keeps on pinging up") def scp_file(self, src_file_path, dst_file_path): self.util.copyFilesToDest(src_file_path, self.bmc_username, self.bmc_ip, dst_file_path, self.bmc_password, "2", BMC_CONST.SCP_TO_REMOTE) def get_version_tar(self, file_path): tar = tarfile.open(file_path) for member in tar.getmembers(): fd = tar.extractfile(member) content = fd.read() if "version=" in content: content = content.split("\n") content = [x for x in content if "version=" in x] version = content[0].split("=")[-1] break tar.close() print version return version def get_image_version(self, path): output = self.cv_BMC.run_command("cat %s | grep \"version=\"" % path) return output[0].split("=")[-1] def delete_images_dir(self): try: self.cv_BMC.run_command("rm -rf /tmp/images/*") except CommandFailed: pass def get_image_path(self, image_version): retry = 0 while (retry < 20): image_list = [] try: image_list = self.cv_BMC.run_command("ls -1 -d /tmp/images/*/ --color=never") except CommandFailed as cf: pass for i in range(0, len(image_list)): version = self.get_image_version(image_list[i] + "MANIFEST") if (version == image_version): return image_list[i] time.sleep(5) retry += 1 def get_image_id(self, version): img_path = self.get_image_path(version) img_id = img_path.split("/")[-2] print "Image id for Host image is : %s" % img_id return img_id def wait_for_bmc_runtime(self): self.util.PingFunc(self.bmc_ip, BMC_CONST.PING_RETRY_FOR_STABILITY) if "SMC" in self.bmc_type: self.cv_IPMI.ipmi_wait_for_bmc_runtime() elif "OpenBMC" in self.bmc_type: self.cv_REST.wait_for_bmc_runtime() return
class OpTestMtdPnorDriver(): ## Initialize this object and also getting the lpar login credentials to use by scp utility # @param i_bmcIP The IP address of the BMC # @param i_bmcUser The userid to log into the BMC with # @param i_bmcPasswd The password of the userid to log into the BMC with # @param i_bmcUserIpmi The userid to issue the BMC IPMI commands with # @param i_bmcPasswdIpmi The password of BMC IPMI userid # @param i_ffdcDir Optional param to indicate where to write FFDC # # "Only required for inband tests" else Default = None # @param i_lparIP The IP address of the LPAR # @param i_lparuser The userid to log into the LPAR # @param i_lparPasswd The password of the userid to log into the LPAR with # def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_lparip=None, i_lparuser=None, i_lparPasswd=None): self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir) self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir) self.cv_LPAR = OpTestLpar(i_lparip, i_lparuser, i_lparPasswd, i_bmcIP) self.util = OpTestUtil() self.lpar_user = i_lparuser self.lpar_ip = i_lparip self.lpar_Passwd = i_lparPasswd ## # @brief This function has following test steps # 1. Get lpar information(OS and Kernel information) # 2. Load the mtd module based on config value # 3. Check /dev/mtd0 character device file existence on lpar # 4. Copying the contents of the flash in a file /tmp/pnor # 5. Getting the /tmp/pnor file into local x86 machine using scp utility # 6. Remove existing /tmp/ffs directory and # Clone latest ffs git repository in local x86 working machine # 7. Compile ffs repository to get fcp utility # 8. Check existence of fcp utility in ffs repository, after compiling # 9. Get the PNOR flash contents on an x86 machine using fcp utility # # @return BMC_CONST.FW_SUCCESS-success or raise OpTestError-fail # def testMtdPnorDriver(self): # Get OS level l_oslevel = self.cv_LPAR.lpar_get_OS_Level() # Get Kernel Version l_kernel = self.cv_LPAR.lpar_get_kernel_version() # loading mtd module based on config option l_config = "CONFIG_MTD_POWERNV_FLASH" l_module = "mtd" self.cv_LPAR.lpar_load_module_based_on_config(l_kernel, l_config, l_module) # Check /dev/mtd0 file existence on lpar l_cmd = "ls -l /dev/mtd0; echo $?" l_res = self.cv_LPAR.lpar_run_command(l_cmd) l_res = l_res.splitlines() if int(l_res[-1]) == 0: print "/dev/mtd0 character device file exists on lpar" else: l_msg = "/dev/mtd0 character device file doesn't exist on lpar" print l_msg raise OpTestError(l_msg) # Copying the contents of the PNOR flash in a file /tmp/pnor l_file = "/tmp/pnor" l_cmd = "cat /dev/mtd0 > %s; echo $?" % l_file l_res = self.cv_LPAR.lpar_run_command(l_cmd) l_res = l_res.splitlines() if int(l_res[-1]) == 0: print "Fetched PNOR data from /dev/mtd0 into temp file /tmp/pnor" else: l_msg = "Fetching PNOR data is failed from /dev/mtd0 into temp file /tmp/pnor" print l_msg raise OpTestError(l_msg) # Getting the /tmp/pnor file into local x86 machine l_path = "/tmp/" self.util.copyFilesToDest(l_path, self.lpar_user, self.lpar_ip, l_file, self.lpar_Passwd, "2", BMC_CONST.SCP_TO_LOCAL) l_list = commands.getstatusoutput("ls -l %s; echo $?" % l_path) print l_list l_workdir = "/tmp/ffs" # Remove existing /tmp/ffs directory l_res = commands.getstatusoutput("rm -rf %s" % l_workdir) print l_res # Clone latest ffs git repository in local x86 working machine l_cmd = "git clone https://github.com/open-power/ffs/ %s" % l_workdir l_res = commands.getstatusoutput(l_cmd) print l_res if int(l_res[0]) == 0: print "Cloning of ffs repository is successfull" else: l_msg = "Cloning ffs repository is failed" print l_msg raise OpTestError(l_msg) # Compile ffs repository to get fcp utility l_cmd = "cd %s/; make" % l_workdir l_res = commands.getstatusoutput(l_cmd) print l_res if int(l_res[0]) == 0: print "Compiling fcp utility is successfull" else: l_msg = "Compiling fcp utility is failed" print l_msg raise OpTestError(l_msg) # Check existence of fcp utility in ffs repository, after compiling. l_cmd = "test -f %s/fcp/x86/fcp" % l_workdir l_res = commands.getstatusoutput(l_cmd) print l_res if int(l_res[0]) == 0: print "Compiling fcp utility is successfull" else: l_msg = "Compiling fcp utility is failed" print l_msg raise OpTestError(l_msg) # Check the PNOR flash contents on an x86 machine using fcp utility l_cmd = "%s/fcp/x86/fcp -o 0x0 -L %s" % (l_workdir, l_file) l_res = commands.getstatusoutput(l_cmd) print l_res[1] if int(l_res[0]) == 0: print "Getting PNOR data successfull using fcp utility" return BMC_CONST.FW_SUCCESS else: l_msg = "Getting the PNOR data using fcp utility failed" print l_msg raise OpTestError(l_msg)