def __init__(self):
# initialize OpTestUtil with this object the OpTestConfiguration
self.util = OpTestUtil(self)
# initialize OpTestCronus with this object the OpTestConfiguration
self.cronus = OpTestCronus(self)
self.args = []
self.remaining_args = []
self.basedir = os.path.dirname(sys.argv[0])
self.signal_ready = False # indicator for properly initialized
self.atexit_ready = False # indicator for properly initialized
self.aes_print_helpers = True # Need state for locker_wait
self.dump = True # Need state for cleanup
self.lock_dict = {'res_id': None,
'name': None,
'Group_Name': None,
'envs': [],
}
self.util_server = None # Hostlocker or AES
self.util_bmc_server = None # OpenBMC REST Server
atexit.register(self.cleanup) # allows cleanup handler to run (OpExit)
self.firmware_versions = None
self.nvram_debug_opts = None
for dirname in (next(os.walk(os.path.join(self.basedir, 'addons')))[1]):
if dirname == "__pycache__":
continue
optAddons[dirname] = importlib.import_module(
"addons." + dirname + ".OpTest" + dirname + "Setup")
return
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_HOST = conf.host()
self.cv_IPMI = conf.ipmi()
self.cv_FSP = conf.bmc()
self.cv_SYSTEM = conf.system()
self.bmc_type = conf.args.bmc_type
self.util = OpTestUtil()
def __init__(self,
hmc_ip,
user_name,
password,
managed_system,
lpar_name,
lpar_vios,
lpar_prof,
lpar_user,
lpar_password,
block_setup_term=None,
delaybeforesend=None,
timeout_factor=1,
logfile=sys.stdout,
prompt=None,
scratch_disk="",
check_ssh_keys=False,
known_hosts_file=None,
proxy=""):
self.logfile = logfile
self.hmc_ip = hmc_ip
self.user = user_name
self.passwd = password
self.mg_system = managed_system
self.util = OpTestUtil()
self.expect_prompt = self.util.build_prompt(prompt) + "$"
self.lpar_name = lpar_name
self.lpar_vios = lpar_vios
self.lpar_prof = lpar_prof
self.lpar_user = lpar_user
self.lpar_password = lpar_password
self.scratch_disk = scratch_disk
self.proxy = proxy
self.state = ConsoleState.DISCONNECTED
self.delaybeforesend = delaybeforesend
self.system = None
# OpTestUtil instance is NOT conf's
self.prompt = prompt
self.pty = None
self.delaybeforesend = delaybeforesend
# allows caller specific control of when to block setup_term
self.block_setup_term = block_setup_term
# tells setup_term to not throw exceptions, like when system off
self.setup_term_quiet = 0
# flags the object to abandon setup_term operations, like when system off
self.setup_term_disable = 0
# FUTURE - System Console currently tracked in System Object
# state tracking, reset on boot and state changes
self.PS1_set = -1
self.LOGIN_set = -1
self.SUDO_set = -1
super(HMCConsole,
self).__init__(hmc_ip, user_name, password, scratch_disk, proxy,
logfile, managed_system, lpar_name, prompt,
block_setup_term, delaybeforesend, timeout_factor,
lpar_prof, lpar_vios, lpar_user, lpar_password,
check_ssh_keys, known_hosts_file)
def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd,
i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_hostip=None,
i_hostuser=None, i_hostPasswd=None):
self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir)
self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi,
i_ffdcDir, i_hostip, i_hostuser, i_hostPasswd)
self.cv_HOST = OpTestHost(i_hostip, i_hostuser, i_hostPasswd, i_bmcIP)
self.cv_SYSTEM = OpTestSystem(i_bmcIP, i_bmcUser, i_bmcPasswd,
i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostip,
i_hostuser, i_hostPasswd)
self.util = OpTestUtil()
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_SYSTEM = conf.system()
self.cv_BMC = conf.bmc()
self.cv_HOST = conf.host()
self.cv_IPMI = conf.ipmi()
self.platform = conf.platform()
self.util = OpTestUtil()
self.bmc_type = conf.args.bmc_type
self.bmc_ip = conf.args.bmc_ip
self.bmc_username = conf.args.bmc_username
self.bmc_password = conf.args.bmc_password
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_SYSTEM = conf.system()
self.cv_BMC = conf.bmc()
self.cv_REST = self.cv_BMC.get_rest_api()
self.cv_HOST = conf.host()
self.cv_IPMI = conf.ipmi()
self.platform = conf.platform()
self.util = OpTestUtil()
self.bmc_type = conf.args.bmc_type
self.bmc_ip = conf.args.bmc_ip
self.bmc_username = conf.args.bmc_username
self.bmc_password = conf.args.bmc_password
self.pupdate_binary = conf.args.pupdate
self.pflash = conf.args.pflash
def __init__(self, hmc_ip, user_name, password, scratch_disk="", proxy="",
logfile=sys.stdout, managed_system=None, lpar_name=None, prompt=None,
block_setup_term=None, delaybeforesend=None, timeout_factor=None,
lpar_prof=None, lpar_vios=None, lpar_user=None, lpar_password=None,
check_ssh_keys=False, known_hosts_file=None, tgt_managed_system=None,
tgt_lpar=None):
self.hmc_ip = hmc_ip
self.user = user_name
self.passwd = password
self.logfile = logfile
self.mg_system = managed_system
self.tgt_mg_system = tgt_managed_system
self.tgt_lpar = tgt_lpar
self.check_ssh_keys = check_ssh_keys
self.known_hosts_file = known_hosts_file
self.lpar_name = lpar_name
self.lpar_prof = lpar_prof
self.lpar_user = lpar_user
self.lpar_password = lpar_password
self.lpar_vios = lpar_vios
self.util = OpTestUtil()
self.prompt = prompt
self.expect_prompt = self.util.build_prompt(prompt) + "$"
self.ssh = OpTestSSH(hmc_ip, user_name, password, logfile=self.logfile,
check_ssh_keys=check_ssh_keys,
known_hosts_file=known_hosts_file,
block_setup_term=block_setup_term)
self.scratch_disk = scratch_disk
self.proxy = proxy
self.scratch_disk_size = None
self.delaybeforesend = delaybeforesend
self.system = None
# OpTestUtil instance is NOT conf's
self.pty = None
# allows caller specific control of when to block setup_term
self.block_setup_term = block_setup_term
# tells setup_term to not throw exceptions, like when system off
self.setup_term_quiet = 0
# flags the object to abandon setup_term operations, like when system off
self.setup_term_disable = 0
# functional simulators are very slow, so multiply all default timeouts by this factor
self.timeout_factor = timeout_factor
# state tracking, reset on boot and state changes
# console tracking done on System object for the system console
self.PS1_set = -1
self.LOGIN_set = -1
self.SUDO_set = -1
def __init__(self):
self.util = OpTestUtil(self) # initialize OpTestUtil with this object the OpTestConfiguration
self.cronus = OpTestCronus(self) # initialize OpTestCronus with this object the OpTestConfiguration
self.args = []
self.remaining_args = []
self.basedir = os.path.dirname(sys.argv[0])
self.signal_ready = False # indicator for properly initialized
self.atexit_ready = False # indicator for properly initialized
self.aes_print_helpers = True # Need state for locker_wait
self.dump = True # Need state for cleanup
self.lock_dict = { 'res_id' : None,
'name' : None,
'Group_Name' : None,
'envs' : [],
}
self.util_server = None # Hostlocker or AES
self.util_bmc_server = None # OpenBMC REST Server
atexit.register(self.__del__) # allows cleanup handler to run (OpExit)
self.firmware_versions = None
for dir in (os.walk(os.path.join(self.basedir, 'addons')).next()[1]):
optAddons[dir] = importlib.import_module("addons." + dir + ".OpTest" + dir + "Setup")
return
def setUpClass(cls):
conf = OpTestConfiguration.conf
cls.cv_HOST = conf.host()
cls.cv_IPMI = conf.ipmi()
cls.cv_SYSTEM = conf.system()
cls.util = OpTestUtil()
cls.test = None
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_HOST = conf.host()
self.cv_IPMI = conf.ipmi()
self.cv_SYSTEM = conf.system()
self.util = OpTestUtil()
self.ppc64cpu_freq_re = re.compile(r"([a-z]+):\s+([\d.]+)")
def setUp(self):
conf = OpTestConfiguration.conf
self.conf = conf
self.cv_HOST = conf.host()
self.cv_IPMI = conf.ipmi()
self.cv_SYSTEM = conf.system()
self.cv_BMC = conf.bmc()
self.util = OpTestUtil()
self.bmc_type = conf.args.bmc_type
if not (self.conf.args.os_repo or self.conf.args.os_cdrom):
self.fail(
"Provide installation media for installation with --os-repo or --os-cdrom"
)
if "qemu" not in self.bmc_type and not (
self.conf.args.host_ip and self.conf.args.host_gateway
and self.conf.args.host_dns and self.conf.args.host_submask
and self.conf.args.host_mac):
self.fail(
"Provide host network details refer, --host-{ip,gateway,dns,submask,mac}"
)
if not (self.conf.args.host_user and self.conf.args.host_password):
self.fail(
"Provide host user details refer, --host-{user,password}")
if not self.cv_HOST.get_scratch_disk():
self.fail(
"Provide proper host disk to install refer, --host-scratch-disk"
)
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_HOST = conf.host()
self.cv_IPMI = conf.ipmi()
self.cv_SYSTEM = conf.system()
self.util = OpTestUtil()
self.platform = conf.platform()
def setUp(self):
conf = OpTestConfiguration.conf
self.host = conf.host()
self.ipmi = conf.ipmi()
self.system = conf.system()
self.util = OpTestUtil()
pass
def setUp(self):
self.conf = OpTestConfiguration.conf
self.host = self.conf.host()
self.ipmi = self.conf.ipmi()
self.system = self.conf.system()
self.bmc = self.conf.bmc()
self.util = OpTestUtil()
self.bmc_type = self.conf.args.bmc_type
if not (self.conf.args.os_repo or self.conf.args.os_cdrom):
self.fail(
"Provide installation media for installation, --os-repo is missing"
)
if not (self.conf.args.host_ip and self.conf.args.host_gateway
and self.conf.args.host_dns and self.conf.args.host_submask
and self.conf.args.host_mac):
self.fail(
"Provide host network details refer, --host-{ip,gateway,dns,submask,mac}"
)
if not (self.conf.args.host_user and self.conf.args.host_password):
self.fail(
"Provide host user details refer, --host-{user,password}")
if not self.conf.args.host_scratch_disk:
self.fail(
"Provide proper host disk to install refer, --host-scratch-disk"
)
def setUp(self):
conf = OpTestConfiguration.conf
self.host = conf.host()
self.ipmi = conf.ipmi()
self.system = conf.system()
self.bmc = conf.bmc()
self.util = OpTestUtil()
self.bmc_type = conf.args.bmc_type
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_HOST = conf.host()
self.cv_IPMI = conf.ipmi()
self.cv_SYSTEM = conf.system()
self.cv_BMC = conf.bmc()
self.util = OpTestUtil()
self.pci_good_data_file = conf.lspci_file()
class OpTestFlashBase(unittest.TestCase):
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_SYSTEM = conf.system()
self.cv_BMC = conf.bmc()
self.cv_HOST = conf.host()
self.cv_IPMI = conf.ipmi()
self.platform = conf.platform()
self.util = OpTestUtil()
self.bmc_type = conf.args.bmc_type
self.bmc_ip = conf.args.bmc_ip
self.bmc_username = conf.args.bmc_username
self.bmc_password = conf.args.bmc_password
def validate_side_activated(self):
l_bmc_side, l_pnor_side = self.cv_IPMI.ipmi_get_side_activated()
self.assertIn(BMC_CONST.PRIMARY_SIDE, l_bmc_side,
"BMC: Primary side is not active")
# TODO force setting of primary side to BIOS Golden side sensor
self.assertIn(BMC_CONST.PRIMARY_SIDE, l_pnor_side,
"PNOR: Primary side is not active")
def get_pnor_level(self):
rc = self.cv_IPMI.ipmi_get_PNOR_level()
print rc
def bmc_down_check(self):
cmd = "ping -c 1 " + self.cv_BMC.host_name + " 1> /dev/null; echo $?"
count = 0
while count < 500:
output = commands.getstatusoutput(cmd)
if output[1] != '0':
print "FSP/BMC Comes down"
break
count = count + 1
time.sleep(2)
else:
self.assertTrue(False, "FSP/BMC keeps on pinging up")
return True
def scp_file(self, src_file_path, dst_file_path):
self.util.copyFilesToDest(src_file_path, self.bmc_username,
self.bmc_ip, dst_file_path,
self.bmc_password, "2",
BMC_CONST.SCP_TO_REMOTE)
def setUp(self):
conf = OpTestConfiguration.conf
self.host = conf.host()
self.ipmi = conf.ipmi()
self.system = conf.system()
self.bmc = conf.bmc()
self.util = OpTestUtil()
self.pci_good_data_file = conf.lspci_file()
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_HOST = conf.host()
self.cv_IPMI = conf.ipmi()
self.cv_SYSTEM = conf.system()
self.cv_BMC = conf.bmc()
self.util = OpTestUtil()
pass
def setUpClass(cls):
conf = OpTestConfiguration.conf
cls.cv_HOST = conf.host()
cls.cv_IPMI = conf.ipmi()
cls.cv_FSP = conf.bmc()
cls.cv_SYSTEM = conf.system()
cls.bmc_type = conf.args.bmc_type
cls.util = OpTestUtil()
def setUpClass(cls):
conf = OpTestConfiguration.conf
cls.cv_HOST = conf.host()
cls.cv_IPMI = conf.ipmi()
cls.cv_SYSTEM = conf.system()
cls.util = OpTestUtil()
# By default test will run on all PHBs/PE's, if one want to skip certain ones mention in this format.
cls.skip_phbs = [] # ['PCI0001', 'PCI0002', 'PCI0003', 'PCI0004', 'PCI0005', 'PCI0030', 'PCI0031', 'PCI0032']
cls.skip_pes = [] # ['0002:00:00.0']
def __init__(self,
hmc_ip,
user_name,
password,
scratch_disk="",
proxy="",
logfile=sys.stdout,
managed_system=None,
lpar_name=None,
prompt=None,
lpar_prof=None,
lpar_vios=None,
lpar_user=None,
lpar_password=None,
check_ssh_keys=False,
known_hosts_file=None):
self.hmc_ip = hmc_ip
self.user = user_name
self.passwd = password
self.logfile = logfile
self.system = managed_system
self.system = managed_system
self.check_ssh_keys = check_ssh_keys
self.known_hosts_file = known_hosts_file
self.lpar_name = lpar_name
self.lpar_prof = lpar_prof
self.lpar_user = lpar_user
self.lpar_password = lpar_password
self.lpar_vios = lpar_vios
self.lpar_con = None
self.vterm = False
self.util = OpTestUtil()
self.prompt = prompt
self.expect_prompt = self.util.build_prompt(prompt) + "$"
self.conf = OpTestConfiguration.conf
self.ssh = OpTestSSH(hmc_ip,
user_name,
password,
logfile=self.logfile,
check_ssh_keys=check_ssh_keys,
known_hosts_file=known_hosts_file)
self.scratch_disk = scratch_disk
self.proxy = proxy
self.scratch_disk_size = None
def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd,
i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_hostip=None,
i_hostuser=None, i_hostPasswd=None):
self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir)
self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi,
i_ffdcDir)
self.cv_HOST = OpTestHost(i_hostip, i_hostuser, i_hostPasswd, i_bmcIP)
self.util = OpTestUtil()
self.host_user = i_hostuser
self.host_ip = i_hostip
self.host_Passwd = i_hostPasswd
def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd,
i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_lparip=None,
i_lparuser=None, i_lparPasswd=None):
self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir)
self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi,
i_ffdcDir)
self.cv_LPAR = OpTestLpar(i_lparip, i_lparuser, i_lparPasswd, i_bmcIP)
self.util = OpTestUtil()
self.lpar_user = i_lparuser
self.lpar_ip = i_lparip
self.lpar_Passwd = i_lparPasswd
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_SYSTEM = conf.system()
self.cv_BMC = conf.bmc()
self.cv_REST = self.cv_BMC.get_rest_api()
self.cv_HOST = conf.host()
self.cv_IPMI = conf.ipmi()
self.platform = conf.platform()
self.util = OpTestUtil()
self.bmc_type = conf.args.bmc_type
self.bmc_ip = conf.args.bmc_ip
self.bmc_username = conf.args.bmc_username
self.bmc_password = conf.args.bmc_password
self.pupdate_binary = conf.args.pupdate
self.pflash = conf.args.pflash
def __init__(self):
self.util = OpTestUtil(self) # initialize OpTestUtil with this object the OpTestConfiguration
self.args = []
self.remaining_args = []
self.basedir = os.path.dirname(sys.argv[0])
self.signal_ready = False # indicator for properly initialized
self.lock_dict = { 'res_id' : None,
'name' : None,
'Group_Name' : None,
'envs' : [],
}
self.util_server = None
for dir in (os.walk(os.path.join(self.basedir, 'addons')).next()[1]):
optAddons[dir] = importlib.import_module("addons." + dir + ".OpTest" + dir + "Setup")
return
class ControlC(unittest.TestCase):
'''
Start a process that does a bunch of console output, and then try and
'control-c' it to stop the process and get a prompt back.
'''
CONTROL = 'c'
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_BMC = conf.bmc()
self.cv_SYSTEM = conf.system()
self.util = OpTestUtil()
self.prompt = self.util.build_prompt()
def cleanup(self):
pass
def runTest(self):
self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)
console = self.cv_BMC.get_host_console()
if (isinstance(self.cv_BMC, OpTestMambo.OpTestMambo)):
raise unittest.SkipTest("Mambo so skipping Control-C tests")
raw_pty = console.get_console()
raw_pty.sendline("find /")
time.sleep(2)
raw_pty.sendcontrol(self.CONTROL)
BMC_DISCONNECT = 'SOL session closed by BMC'
timeout = 60
try:
rc = raw_pty.expect([BMC_DISCONNECT, self.prompt,
pexpect.TIMEOUT, pexpect.EOF], timeout)
if rc == 0:
raise BMCDisconnected(BMC_DISCONNECT)
self.assertEqual(rc, 1, "Failed to find expected prompt")
except pexpect.TIMEOUT as e:
log.debug(e)
log.debug("# TIMEOUT waiting for command to finish with ctrl-c.")
log.debug("# Everything is terrible. Fail the world, "
"power cycle (if lucky)")
self.cv_SYSTEM.set_state(OpSystemState.UNKNOWN_BAD)
self.fail("Could not ctrl-c running command in reasonable time")
self.cleanup()
class ControlC(unittest.TestCase):
CONTROL = 'c'
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_BMC = conf.bmc()
self.cv_SYSTEM = conf.system()
self.util = OpTestUtil()
self.prompt = self.util.build_prompt()
def cleanup(self):
pass
def runTest(self):
self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)
console = self.cv_BMC.get_host_console()
# I should really make this API less nasty...
raw_console = console.get_console()
#raw_console.sendline("hexdump -C -v /dev/zero")
raw_console.sendline("find /")
time.sleep(2)
raw_console.sendcontrol(self.CONTROL)
BMC_DISCONNECT = 'SOL session closed by BMC'
timeout = 60
try:
rc = raw_console.expect(
[BMC_DISCONNECT, self.prompt, pexpect.TIMEOUT, pexpect.EOF],
timeout)
if rc == 0:
raise BMCDisconnected(BMC_DISCONNECT)
self.assertEqual(rc, 1, "Failed to find expected prompt")
except pexpect.TIMEOUT as e:
log.debug(e)
log.debug("# TIMEOUT waiting for command to finish with ctrl-c.")
log.debug(
"# Everything is terrible. Fail the world, power cycle (if lucky)"
)
self.cv_SYSTEM.set_state(OpSystemState.UNKNOWN_BAD)
self.fail("Could not ctrl-c running command in reasonable time")
self.cleanup()
class OpTestConfiguration():
def __init__(self):
self.util = OpTestUtil(self) # initialize OpTestUtil with this object the OpTestConfiguration
self.args = []
self.remaining_args = []
self.basedir = os.path.dirname(sys.argv[0])
self.signal_ready = False # indicator for properly initialized
self.atexit_ready = False # indicator for properly initialized
self.aes_print_helpers = True # Need state for locker_wait
self.dump = True # Need state for cleanup
self.lock_dict = { 'res_id' : None,
'name' : None,
'Group_Name' : None,
'envs' : [],
}
self.util_server = None # Hostlocker or AES
self.util_bmc_server = None # OpenBMC REST Server
atexit.register(self.__del__) # allows cleanup handler to run (OpExit)
self.firmware_versions = None
for dir in (os.walk(os.path.join(self.basedir, 'addons')).next()[1]):
optAddons[dir] = importlib.import_module("addons." + dir + ".OpTest" + dir + "Setup")
return
def __del__(self):
if self.atexit_ready:
# calling cleanup before args initialized pointless
# attribute errors thrown in cleanup, e.g. ./op-test -h
self.util.cleanup()
def parse_args(self, argv=None):
conf_parser = argparse.ArgumentParser(add_help=False)
# We have two parsers so we have correct --help, we need -c in both
conf_parser.add_argument("-c", "--config-file", help="Configuration File",
metavar="FILE")
args , remaining_args = conf_parser.parse_known_args(argv)
defaults = {}
config = ConfigParser.SafeConfigParser()
config.read([os.path.expanduser("~/.op-test-framework.conf")])
if args.config_file:
if os.access(args.config_file, os.R_OK):
config.read([args.config_file])
else:
raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), args.config_file)
try:
defaults = dict(config.items('op-test'))
except ConfigParser.NoSectionError:
pass
parser = get_parser()
parser.set_defaults(**defaults)
if defaults.get('qemu_binary'):
qemu_default = defaults['qemu_binary']
if defaults.get('mambo_binary'):
mambo_default = defaults['mambo_binary']
if defaults.get('mambo_initial_run_script'):
mambo_default = defaults['mambo_initial_run_script']
parser.add_argument("--check-ssh-keys", action='store_true', default=False,
help="Check remote host keys when using SSH (auto-yes on new)")
parser.add_argument("--known-hosts-file",
help="Specify a custom known_hosts file")
self.args , self.remaining_args = parser.parse_known_args(remaining_args)
args_dict = vars(self.args)
# if we have a bmc_type we start with appropriate template
if args_dict.get('bmc_type') is not None:
dict_merge = default_templates.get(args_dict.get('bmc_type').lower())
if dict_merge is not None:
default_val.update(dict_merge) # overlays dict merge on top of default_val
for key in default_val:
if args_dict.get(key) is None:
args_dict[key] = default_val[key]
stateMap = { 'UNKNOWN' : common.OpTestSystem.OpSystemState.UNKNOWN,
'UNKNOWN_BAD' : common.OpTestSystem.OpSystemState.UNKNOWN_BAD,
'OFF' : common.OpTestSystem.OpSystemState.OFF,
'PETITBOOT' : common.OpTestSystem.OpSystemState.PETITBOOT,
'PETITBOOT_SHELL' : common.OpTestSystem.OpSystemState.PETITBOOT_SHELL,
'OS' : common.OpTestSystem.OpSystemState.OS
}
# Some quick sanity checking
if self.args.known_hosts_file and not self.args.check_ssh_keys:
parser.error("--known-hosts-file requires --check-ssh-keys")
# Setup some defaults for the output options
# Order of precedence
# 1. cmdline arg
# 2. env variable
# 3. default path
if (self.args.output):
outdir = self.args.output
elif ("OP_TEST_OUTPUT" in os.environ):
outdir = os.environ["OP_TEST_OUTPUT"]
else:
outdir = os.path.join(self.basedir, "test-reports")
self.outsuffix = "test-run-%s" % self.get_suffix()
outdir = os.path.join(outdir, self.outsuffix)
# Normalize the path to fully qualified and create if not there
self.output = os.path.abspath(outdir)
if (not os.path.exists(self.output)):
os.makedirs(self.output)
if (self.args.logdir):
logdir = self.args.logdir
elif ("OP_TEST_LOGDIR" in os.environ):
logdir = os.environ["OP_TEST_LOGDIR"]
else:
logdir = self.output
self.logdir = os.path.abspath(logdir)
if (not os.path.exists(self.logdir)):
os.makedirs(self.logdir)
print("Logs in: {}".format(self.logdir))
OpTestLogger.optest_logger_glob.logdir = self.logdir
# Grab the suffix, if not given use current time
self.outsuffix = self.get_suffix()
# set up where all the logs go
logfile = os.path.join(self.output, "%s.log" % self.outsuffix)
logcmd = "tee %s" % (logfile)
# we use 'cat -v' to convert control characters
# to something that won't affect the user's terminal
if self.args.quiet:
logcmd = logcmd + "> /dev/null"
# save sh_level for later refresh loggers
OpTestLogger.optest_logger_glob.sh_level = logging.ERROR
OpTestLogger.optest_logger_glob.sh.setLevel(logging.ERROR)
else:
logcmd = logcmd + "| sed -u -e 's/\\r$//g'|cat -v"
# save sh_level for later refresh loggers
OpTestLogger.optest_logger_glob.sh_level = logging.INFO
OpTestLogger.optest_logger_glob.sh.setLevel(logging.INFO)
OpTestLogger.optest_logger_glob.setUpLoggerFile(datetime.utcnow().strftime("%Y%m%d%H%M%S%f")+'.main.log')
OpTestLogger.optest_logger_glob.setUpLoggerDebugFile(datetime.utcnow().strftime("%Y%m%d%H%M%S%f")+'.debug.log')
OpTestLogger.optest_logger_glob.optest_logger.info('TestCase Log files: {}/*{}*'.format(self.output, self.outsuffix))
OpTestLogger.optest_logger_glob.optest_logger.info('StreamHandler setup {}'.format('quiet' if self.args.quiet else 'normal'))
self.logfile_proc = subprocess.Popen(logcmd,
stdin=subprocess.PIPE,
stderr=sys.stderr,
stdout=sys.stdout,
shell=True)
self.logfile = self.logfile_proc.stdin
# we have enough setup to allow
# signal handler cleanup to run
self.signal_ready = True
# atexit viable for cleanup to run
self.atexit_ready = True
# now that we have loggers, dump conf file to help debug later
OpTestLogger.optest_logger_glob.optest_logger.debug(
"conf file defaults={}".format(defaults))
# setup AES and Hostlocker configs after the logging is setup
locker_timeout = time.time() + 60*self.args.locker_wait
locker_code = errno.ETIME # 62
locker_message = ("OpTestSystem waited {} minutes but was unable"
" to lock environment/host requested,"
" either pick another environment/host or increase "
"--locker-wait, try --aes q with options for "
"--aes-search-args to view availability, or as"
" appropriate for your hostlocker"
.format(self.args.locker_wait))
locker_exit_exception = OpExit(message=locker_message,
code=locker_code)
while True:
try:
rollup_flag = False
self.util.check_lockers()
break
except Exception as e:
OpTestLogger.optest_logger_glob.optest_logger.debug(
"locker_wait Exception={}".format(e))
if "unable to lock" in e.message:
self.aes_print_helpers = False
# SystemExit exception needs message to print
rollup_message = locker_exit_exception.message
rollup_exception = locker_exit_exception
else:
rollup_message = e.message
rollup_exception = e
rollup_flag = True # bubble exception out
if time.time() > locker_timeout or rollup_flag:
# if not "unable to lock" we bubble up underlying exception
OpTestLogger.optest_logger_glob.optest_logger.warning(
"{}".format(rollup_message))
raise rollup_exception
else:
OpTestLogger.optest_logger_glob.optest_logger.info(
"OpTestSystem waiting for requested environment/host"
" total time to wait is {} minutes, we will check"
" every minute"
.format(self.args.locker_wait))
time.sleep(60)
if self.args.machine_state == None:
if self.args.bmc_type in ['qemu', 'mambo']:
# Force UNKNOWN_BAD so that we don't try to setup the console early
self.startState = common.OpTestSystem.OpSystemState.UNKNOWN_BAD
else:
self.startState = common.OpTestSystem.OpSystemState.UNKNOWN
else:
self.startState = stateMap[self.args.machine_state]
return self.args, self.remaining_args
def get_suffix(self):
# Grab the suffix, if not given use current time
if (self.args.suffix):
outsuffix = self.args.suffix
else:
outsuffix = time.strftime("%Y%m%d%H%M%S")
return outsuffix
def objs(self):
if self.args.list_suites or self.args.list_tests:
return
# check to see if bmc_ip even pings to validate configuration parms
try:
self.util.PingFunc(self.args.bmc_ip, totalSleepTime=BMC_CONST.PING_RETRY_FOR_STABILITY)
except Exception as e:
# we are trying to catch sooner rather than later
# if we have reservations that need cleaned up
# otherwise we would have to try/except for cleanup
# in lots of places
# testcases.HelloWorld in CI fails if we throw this
# raise only if we have reservations to cleanup
if self.args.hostlocker is not None \
or self.args.aes is not None \
or self.args.aes_search_args is not None:
self.util.cleanup()
raise ParameterCheck(message="OpTestSystem PingFunc fails to "
"ping '{}', check your configuration and setup, see "
"Exception details: {}".format(self.args.bmc_ip, e))
try:
host = common.OpTestHost.OpTestHost(self.args.host_ip,
self.args.host_user,
self.args.host_password,
self.args.bmc_ip,
self.output,
scratch_disk=self.args.host_scratch_disk,
proxy=self.args.proxy,
check_ssh_keys=self.args.check_ssh_keys,
known_hosts_file=self.args.known_hosts_file,
conf=self)
if self.args.bmc_type in ['AMI', 'SMC']:
web = OpTestWeb(self.args.bmc_ip,
self.args.bmc_usernameipmi,
self.args.bmc_passwordipmi)
bmc = None
if self.args.bmc_type in ['AMI']:
ipmi = OpTestIPMI(self.args.bmc_ip,
self.args.bmc_usernameipmi,
self.args.bmc_passwordipmi,
host=host,
logfile=self.logfile,
)
bmc = OpTestBMC(ip=self.args.bmc_ip,
username=self.args.bmc_username,
password=self.args.bmc_password,
logfile=self.logfile,
ipmi=ipmi,
web=web,
check_ssh_keys=self.args.check_ssh_keys,
known_hosts_file=self.args.known_hosts_file
)
elif self.args.bmc_type in ['SMC']:
ipmi = OpTestSMCIPMI(self.args.bmc_ip,
self.args.bmc_usernameipmi,
self.args.bmc_passwordipmi,
logfile=self.logfile,
host=host,
)
bmc = OpTestSMC(ip=self.args.bmc_ip,
username=self.args.bmc_username,
password=self.args.bmc_password,
ipmi=ipmi,
web=web,
check_ssh_keys=self.args.check_ssh_keys,
known_hosts_file=self.args.known_hosts_file
)
self.op_system = common.OpTestSystem.OpTestSystem(
state=self.startState,
bmc=bmc,
host=host,
conf=self,
)
ipmi.set_system(self.op_system)
bmc.set_system(self.op_system)
elif self.args.bmc_type in ['FSP']:
ipmi = OpTestIPMI(self.args.bmc_ip,
None, # FSP does not use UID
self.args.bmc_passwordipmi,
host=host,
logfile=self.logfile)
bmc = OpTestFSP(self.args.bmc_ip,
self.args.bmc_username,
self.args.bmc_password,
ipmi=ipmi,
)
self.op_system = common.OpTestSystem.OpTestFSPSystem(
state=self.startState,
bmc=bmc,
host=host,
conf=self,
)
ipmi.set_system(self.op_system)
elif self.args.bmc_type in ['OpenBMC']:
ipmi = OpTestIPMI(self.args.bmc_ip,
self.args.bmc_usernameipmi,
self.args.bmc_passwordipmi,
host=host,
logfile=self.logfile)
rest_api = HostManagement(conf=self,
ip=self.args.bmc_ip,
username=self.args.bmc_username,
password=self.args.bmc_password)
bmc = OpTestOpenBMC(ip=self.args.bmc_ip,
username=self.args.bmc_username,
password=self.args.bmc_password,
ipmi=ipmi,
rest_api=rest_api,
logfile=self.logfile,
check_ssh_keys=self.args.check_ssh_keys,
known_hosts_file=self.args.known_hosts_file)
self.op_system = common.OpTestSystem.OpTestOpenBMCSystem(
host=host,
bmc=bmc,
state=self.startState,
conf=self,
)
bmc.set_system(self.op_system)
elif self.args.bmc_type in ['qemu']:
print(repr(self.args))
bmc = OpTestQemu(qemu_binary=self.args.qemu_binary,
pnor=self.args.host_pnor,
skiboot=self.args.flash_skiboot,
kernel=self.args.flash_kernel,
initramfs=self.args.flash_initramfs,
cdrom=self.args.os_cdrom,
logfile=self.logfile,
hda=self.args.host_scratch_disk)
self.op_system = common.OpTestSystem.OpTestQemuSystem(host=host,
bmc=bmc,
state=self.startState,
conf=self,
)
bmc.set_system(self.op_system)
elif self.args.bmc_type in ['mambo']:
if not (os.stat(self.args.mambo_binary).st_mode & stat.S_IXOTH):
raise ParameterCheck(message="Check that the file exists with"
" X permissions"
" mambo-binary={}"
.format(self.args.mambo_binary))
if self.args.flash_skiboot is None \
or not os.access(self.args.flash_skiboot, os.R_OK|os.W_OK):
raise ParameterCheck(message="Check that the file exists with"
" R/W/X permissions"
" flash-skiboot={}"
.format(self.args.flash_skiboot))
if self.args.flash_kernel is None \
or not os.access(self.args.flash_kernel, os.R_OK|os.W_OK):
raise ParameterCheck(message="Check that the file exists with"
" R/W permissions"
" flash-kernel={}"
.format(self.args.flash_kernel))
bmc = OpTestMambo(mambo_binary=self.args.mambo_binary,
mambo_initial_run_script=self.args.mambo_initial_run_script,
mambo_autorun=self.args.mambo_autorun,
skiboot=self.args.flash_skiboot,
kernel=self.args.flash_kernel,
initramfs=self.args.flash_initramfs,
logfile=self.logfile)
self.op_system = common.OpTestSystem.OpTestMamboSystem(host=host,
bmc=bmc,
state=self.startState,
conf=self,
)
bmc.set_system(self.op_system)
# Check that the bmc_type exists in our loaded addons then create our objects
elif self.args.bmc_type in optAddons:
(bmc, self.op_system) = optAddons[self.args.bmc_type].createSystem(self, host)
else:
self.util.cleanup()
raise Exception("Unsupported BMC Type '{}', check your "
"upper/lower cases for bmc_type and verify "
"any credentials used from HostLocker or "
"AES Version (see aes_get_creds "
"version_mappings)".format(self.args.bmc_type))
host.set_system(self.op_system)
return
except Exception as e:
traceback.print_exc()
self.util.cleanup()
raise e
def bmc(self):
return self.op_system.bmc
def system(self):
return self.op_system
def host(self):
return self.op_system.host()
def ipmi(self):
return self.op_system.ipmi()
def lspci_file(self):
return self.args.host_lspci
def platform(self):
return self.args.platform
class OpTestConfiguration():
def __init__(self):
# initialize OpTestUtil with this object the OpTestConfiguration
self.util = OpTestUtil(self)
# initialize OpTestCronus with this object the OpTestConfiguration
self.cronus = OpTestCronus(self)
self.args = []
self.remaining_args = []
self.basedir = os.path.dirname(sys.argv[0])
self.signal_ready = False # indicator for properly initialized
self.atexit_ready = False # indicator for properly initialized
self.aes_print_helpers = True # Need state for locker_wait
self.dump = True # Need state for cleanup
self.lock_dict = {'res_id': None,
'name': None,
'Group_Name': None,
'envs': [],
}
self.util_server = None # Hostlocker or AES
self.util_bmc_server = None # OpenBMC REST Server
atexit.register(self.cleanup) # allows cleanup handler to run (OpExit)
self.firmware_versions = None
self.nvram_debug_opts = None
for dirname in (next(os.walk(os.path.join(self.basedir, 'addons')))[1]):
if dirname == "__pycache__":
continue
optAddons[dirname] = importlib.import_module(
"addons." + dirname + ".OpTest" + dirname + "Setup")
return
def cleanup(self):
if self.atexit_ready:
# calling cleanup before args initialized pointless
# attribute errors thrown in cleanup, e.g. ./op-test -h
self.util.cleanup()
def parse_args(self, argv=None):
conf_parser = argparse.ArgumentParser(add_help=False)
# We have two parsers so we have correct --help, we need -c in both
conf_parser.add_argument("-c", "--config-file", help="Configuration File",
metavar="FILE")
args, remaining_args = conf_parser.parse_known_args(argv)
defaults = {}
config = configparser.SafeConfigParser()
config.read([os.path.expanduser("~/.op-test-framework.conf")])
if args.config_file:
if os.access(args.config_file, os.R_OK):
config.read([args.config_file])
else:
raise OSError(errno.ENOENT, os.strerror(
errno.ENOENT), args.config_file)
try:
defaults = dict(config.items('op-test'))
except configparser.NoSectionError:
pass
parser = get_parser()
parser.set_defaults(**defaults)
if defaults.get('qemu_binary'):
qemu_default = defaults['qemu_binary']
if defaults.get('mambo_binary'):
mambo_default = defaults['mambo_binary']
if defaults.get('mambo_initial_run_script'):
mambo_default = defaults['mambo_initial_run_script']
parser.add_argument("--check-ssh-keys", action='store_true', default=False,
help="Check remote host keys when using SSH (auto-yes on new)")
parser.add_argument("--known-hosts-file",
help="Specify a custom known_hosts file")
self.args, self.remaining_args = parser.parse_known_args(
remaining_args)
args_dict = vars(self.args)
# if we have a bmc_type we start with appropriate template
if args_dict.get('bmc_type') is not None:
dict_merge = default_templates.get(
args_dict.get('bmc_type').lower())
if dict_merge is not None:
# overlays dict merge on top of default_val
default_val.update(dict_merge)
for key in default_val:
if args_dict.get(key) is None:
args_dict[key] = default_val[key]
stateMap = {'UNKNOWN': common.OpTestSystem.OpSystemState.UNKNOWN,
'UNKNOWN_BAD': common.OpTestSystem.OpSystemState.UNKNOWN_BAD,
'OFF': common.OpTestSystem.OpSystemState.OFF,
'PETITBOOT': common.OpTestSystem.OpSystemState.PETITBOOT,
'PETITBOOT_SHELL': common.OpTestSystem.OpSystemState.PETITBOOT_SHELL,
'OS': common.OpTestSystem.OpSystemState.OS
}
# Some quick sanity checking
if self.args.known_hosts_file and not self.args.check_ssh_keys:
parser.error("--known-hosts-file requires --check-ssh-keys")
# Setup some defaults for the output options
# Order of precedence
# 1. cmdline arg
# 2. env variable
# 3. default path
if (self.args.output):
outdir = self.args.output
elif ("OP_TEST_OUTPUT" in os.environ):
outdir = os.environ["OP_TEST_OUTPUT"]
else:
outdir = os.path.join(self.basedir, "test-reports")
self.outsuffix = "test-run-%s" % self.get_suffix()
outdir = os.path.join(outdir, self.outsuffix)
# Normalize the path to fully qualified and create if not there
self.output = os.path.abspath(outdir)
if (not os.path.exists(self.output)):
os.makedirs(self.output)
if (self.args.logdir):
logdir = self.args.logdir
elif ("OP_TEST_LOGDIR" in os.environ):
logdir = os.environ["OP_TEST_LOGDIR"]
else:
logdir = self.output
self.logdir = os.path.abspath(logdir)
if (not os.path.exists(self.logdir)):
os.makedirs(self.logdir)
print(("Logs in: {}".format(self.logdir)))
OpTestLogger.optest_logger_glob.logdir = self.logdir
# Grab the suffix, if not given use current time
self.outsuffix = self.get_suffix()
# set up where all the logs go
logfile = os.path.join(self.output, "%s.log" % self.outsuffix)
logcmd = "tee %s" % (logfile)
# we use 'cat -v' to convert control characters
# to something that won't affect the user's terminal
if self.args.quiet:
logcmd = logcmd + "> /dev/null"
# save sh_level for later refresh loggers
OpTestLogger.optest_logger_glob.sh_level = logging.ERROR
OpTestLogger.optest_logger_glob.sh.setLevel(logging.ERROR)
else:
logcmd = logcmd + "| sed -u -e 's/\\r$//g'|cat -v"
# save sh_level for later refresh loggers
OpTestLogger.optest_logger_glob.sh_level = logging.INFO
OpTestLogger.optest_logger_glob.sh.setLevel(logging.INFO)
OpTestLogger.optest_logger_glob.setUpLoggerFile(
datetime.utcnow().strftime("%Y%m%d%H%M%S%f")+'.main.log')
OpTestLogger.optest_logger_glob.setUpLoggerDebugFile(
datetime.utcnow().strftime("%Y%m%d%H%M%S%f")+'.debug.log')
OpTestLogger.optest_logger_glob.optest_logger.info(
'TestCase Log files: {}/*'.format(self.output))
OpTestLogger.optest_logger_glob.optest_logger.info(
'StreamHandler setup {}'.format('quiet' if self.args.quiet else 'normal'))
self.logfile_proc = subprocess.Popen(logcmd,
stdin=subprocess.PIPE,
stderr=sys.stderr,
stdout=sys.stdout,
shell=True,
universal_newlines=True,
encoding='utf-8')
self.logfile = self.logfile_proc.stdin
# we have enough setup to allow
# signal handler cleanup to run
self.signal_ready = True
# atexit viable for cleanup to run
self.atexit_ready = True
# now that we have loggers, dump conf file to help debug later
OpTestLogger.optest_logger_glob.optest_logger.debug(
"conf file defaults={}".format(defaults))
cmd = "git describe --always"
try:
git_output = subprocess.check_output(cmd.split())
# log for triage of how dated the repo is
OpTestLogger.optest_logger_glob.optest_logger.debug(
"op-test-framework git level = {}".format(git_output))
except Exception as e:
OpTestLogger.optest_logger_glob.optest_logger.debug(
"Unable to get git describe")
# setup AES and Hostlocker configs after the logging is setup
locker_timeout = time.time() + 60*self.args.locker_wait
locker_code = errno.ETIME # 62
locker_message = ("OpTestSystem waited {} minutes but was unable"
" to lock environment/host requested,"
" either pick another environment/host or increase "
"--locker-wait, try --aes q with options for "
"--aes-search-args to view availability, or as"
" appropriate for your hostlocker"
.format(self.args.locker_wait))
locker_exit_exception = OpExit(message=locker_message,
code=locker_code)
while True:
try:
rollup_flag = False
self.util.check_lockers()
break
except Exception as e:
OpTestLogger.optest_logger_glob.optest_logger.debug(
"locker_wait Exception={}".format(e))
if "unable to lock" in e.message:
self.aes_print_helpers = False
# SystemExit exception needs message to print
rollup_message = locker_exit_exception.message
rollup_exception = locker_exit_exception
else:
rollup_message = e.message
rollup_exception = e
rollup_flag = True # bubble exception out
if time.time() > locker_timeout or rollup_flag:
# if not "unable to lock" we bubble up underlying exception
OpTestLogger.optest_logger_glob.optest_logger.warning(
"{}".format(rollup_message))
raise rollup_exception
else:
OpTestLogger.optest_logger_glob.optest_logger.info(
"OpTestSystem waiting for requested environment/host"
" total time to wait is {} minutes, we will check"
" every minute"
.format(self.args.locker_wait))
time.sleep(60)
if self.args.machine_state == None:
if self.args.bmc_type in ['qemu', 'mambo']:
# Force UNKNOWN_BAD so that we don't try to setup the console early
self.startState = common.OpTestSystem.OpSystemState.UNKNOWN_BAD
else:
self.startState = common.OpTestSystem.OpSystemState.UNKNOWN
else:
self.startState = stateMap[self.args.machine_state]
return self.args, self.remaining_args
def get_suffix(self):
# Grab the suffix, if not given use current time
if (self.args.suffix):
outsuffix = self.args.suffix
else:
outsuffix = time.strftime("%Y%m%d%H%M%S")
return outsuffix
def objs(self):
if self.args.list_suites or self.args.list_tests:
return
# check to see if bmc_ip even pings to validate configuration parms
try:
self.util.PingFunc(
self.args.bmc_ip, totalSleepTime=BMC_CONST.PING_RETRY_FOR_STABILITY)
except Exception as e:
# we are trying to catch sooner rather than later
# if we have reservations that need cleaned up
# otherwise we would have to try/except for cleanup
# in lots of places
# testcases.HelloWorld in CI fails if we throw this
# raise only if we have reservations to cleanup
if self.args.hostlocker is not None \
or self.args.aes is not None \
or self.args.aes_search_args is not None:
self.util.cleanup()
raise ParameterCheck(message="OpTestSystem PingFunc fails to "
"ping '{}', check your configuration and setup, see "
"Exception details: {}".format(self.args.bmc_ip, e))
try:
host = common.OpTestHost.OpTestHost(self.args.host_ip,
self.args.host_user,
self.args.host_password,
self.args.bmc_ip,
self.output,
scratch_disk=self.args.host_scratch_disk,
proxy=self.args.proxy,
logfile=self.logfile,
check_ssh_keys=self.args.check_ssh_keys,
known_hosts_file=self.args.known_hosts_file,
conf=self)
if self.args.bmc_type in ['AMI', 'SMC']:
web = OpTestWeb(self.args.bmc_ip,
self.args.bmc_usernameipmi,
self.args.bmc_passwordipmi)
bmc = None
if self.args.bmc_type in ['AMI']:
ipmi = OpTestIPMI(self.args.bmc_ip,
self.args.bmc_usernameipmi,
self.args.bmc_passwordipmi,
host=host,
host_console_command=self.args.host_serial_console_command,
logfile=self.logfile,
)
bmc = OpTestBMC(ip=self.args.bmc_ip,
username=self.args.bmc_username,
password=self.args.bmc_password,
logfile=self.logfile,
ipmi=ipmi,
web=web,
check_ssh_keys=self.args.check_ssh_keys,
known_hosts_file=self.args.known_hosts_file
)
elif self.args.bmc_type in ['SMC']:
ipmi = OpTestSMCIPMI(self.args.bmc_ip,
self.args.bmc_usernameipmi,
self.args.bmc_passwordipmi,
logfile=self.logfile,
host=host,
)
bmc = OpTestSMC(ip=self.args.bmc_ip,
username=self.args.bmc_username,
password=self.args.bmc_password,
ipmi=ipmi,
web=web,
check_ssh_keys=self.args.check_ssh_keys,
known_hosts_file=self.args.known_hosts_file
)
self.op_system = common.OpTestSystem.OpTestSystem(
state=self.startState,
bmc=bmc,
host=host,
conf=self,
)
ipmi.set_system(self.op_system)
bmc.set_system(self.op_system)
elif self.args.bmc_type in ['FSP']:
ipmi = OpTestIPMI(self.args.bmc_ip,
None, # FSP does not use UID
self.args.bmc_passwordipmi,
host=host,
logfile=self.logfile)
bmc = OpTestFSP(self.args.bmc_ip,
self.args.bmc_username,
self.args.bmc_password,
ipmi=ipmi,
)
self.op_system = common.OpTestSystem.OpTestFSPSystem(
state=self.startState,
bmc=bmc,
host=host,
conf=self,
)
ipmi.set_system(self.op_system)
elif self.args.bmc_type in ['FSP_PHYP']:
hmc = None
if all(v is not None for v in [self.args.hmc_ip, self.args.hmc_username, self.args.hmc_password]):
hmc = OpTestHMC(self.args.hmc_ip,
self.args.hmc_username,
self.args.hmc_password,
managed_system=self.args.system_name,
lpar_name=self.args.lpar_name,
lpar_vios=self.args.lpar_vios,
lpar_prof=self.args.lpar_prof,
lpar_user=self.args.host_user,
lpar_password=self.args.host_password,
logfile=self.logfile
)
else:
raise Exception(
"HMC IP, username and password is required")
bmc = OpTestFSP(self.args.bmc_ip,
self.args.bmc_username,
self.args.bmc_password,
)
self.op_system = common.OpTestSystem.OpTestFSPSystem(
state=self.startState,
bmc=bmc,
host=host,
hmc=hmc,
conf=self,
)
hmc.set_system(self.op_system)
elif self.args.bmc_type in ['OpenBMC']:
ipmi = OpTestIPMI(self.args.bmc_ip,
self.args.bmc_usernameipmi,
self.args.bmc_passwordipmi,
host=host,
logfile=self.logfile)
rest_api = HostManagement(conf=self,
ip=self.args.bmc_ip,
username=self.args.bmc_username,
password=self.args.bmc_password)
bmc = OpTestOpenBMC(ip=self.args.bmc_ip,
username=self.args.bmc_username,
password=self.args.bmc_password,
ipmi=ipmi,
rest_api=rest_api,
logfile=self.logfile,
check_ssh_keys=self.args.check_ssh_keys,
known_hosts_file=self.args.known_hosts_file)
self.op_system = common.OpTestSystem.OpTestOpenBMCSystem(
host=host,
bmc=bmc,
state=self.startState,
conf=self,
)
bmc.set_system(self.op_system)
elif self.args.bmc_type in ['qemu']:
print((repr(self.args)))
bmc = OpTestQemu(conf=self,
qemu_binary=self.args.qemu_binary,
pnor=self.args.host_pnor,
skiboot=self.args.flash_skiboot,
kernel=self.args.flash_kernel,
initramfs=self.args.flash_initramfs,
cdrom=self.args.os_cdrom,
logfile=self.logfile)
self.op_system = common.OpTestSystem.OpTestQemuSystem(host=host,
bmc=bmc,
state=self.startState,
conf=self,
)
bmc.set_system(self.op_system)
elif self.args.bmc_type in ['mambo']:
if not (os.stat(self.args.mambo_binary).st_mode & stat.S_IXOTH):
raise ParameterCheck(message="Check that the file exists with X permissions mambo-binary={}"
.format(self.args.mambo_binary))
if self.args.flash_skiboot is None \
or not os.access(self.args.flash_skiboot, os.R_OK):
raise ParameterCheck(message="Check that the file exists with R permissions flash-skiboot={}"
.format(self.args.flash_skiboot))
if self.args.flash_kernel is None \
or not os.access(self.args.flash_kernel, os.R_OK):
raise ParameterCheck(message="Check that the file exists with R permissions flash-kernel={}"
.format(self.args.flash_kernel))
bmc = OpTestMambo(mambo_binary=self.args.mambo_binary,
mambo_initial_run_script=self.args.mambo_initial_run_script,
mambo_autorun=self.args.mambo_autorun,
skiboot=self.args.flash_skiboot,
kernel=self.args.flash_kernel,
initramfs=self.args.flash_initramfs,
timeout_factor=self.args.mambo_timeout_factor,
logfile=self.logfile)
self.op_system = common.OpTestSystem.OpTestMamboSystem(host=host,
bmc=bmc,
state=self.startState,
conf=self,
)
bmc.set_system(self.op_system)
# Check that the bmc_type exists in our loaded addons then create our objects
elif self.args.bmc_type in optAddons:
(bmc, self.op_system) = optAddons[self.args.bmc_type].createSystem(
self, host)
else:
self.util.cleanup()
raise Exception("Unsupported BMC Type '{}', check your "
"upper/lower cases for bmc_type and verify "
"any credentials used from HostLocker or "
"AES Version (see aes_get_creds "
"version_mappings)".format(self.args.bmc_type))
host.set_system(self.op_system)
return
except Exception as e:
traceback.print_exc()
self.util.cleanup()
raise e
def bmc(self):
return self.op_system.bmc
def hmc(self):
return self.op_system.hmc
def system(self):
return self.op_system
def host(self):
return self.op_system.host()
def ipmi(self):
return self.op_system.ipmi()
def lspci_file(self):
return self.args.host_lspci
def platform(self):
return self.args.platform
class OpTestIPMIReprovision():
## Initialize this object
# @param i_bmcIP The IP address of the BMC
# @param i_bmcUser The userid to log into the BMC with
# @param i_bmcPasswd The password of the userid to log into the BMC with
# @param i_bmcUserIpmi The userid to issue the BMC IPMI commands with
# @param i_bmcPasswdIpmi The password of BMC IPMI userid
# @param i_ffdcDir Optional param to indicate where to write FFDC
#
# "Only required for inband tests" else Default = None
# @param i_hostIP The IP address of the HOST
# @param i_hostUser The userid to log into the HOST
# @param i_hostPasswd The password of the userid to log into the HOST with
#
def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd,
i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_hostIP=None,
i_hostUser=None, i_hostPasswd=None):
self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir)
self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi,
i_ffdcDir, i_hostIP, i_hostUser, i_hostPasswd)
self.cv_HOST = OpTestHost(i_hostIP, i_hostUser, i_hostPasswd,i_bmcIP, i_ffdcDir)
self.cv_SYSTEM = OpTestSystem(i_bmcIP, i_bmcUser, i_bmcPasswd,
i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostIP,
i_hostUser, i_hostPasswd)
self.util = OpTestUtil()
self.opTestHMIHandling = OpTestHMIHandling(i_bmcIP, i_bmcUser, i_bmcPasswd,
i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostIP,
i_hostUser, i_hostPasswd)
##
# @brief This function will cover following test steps
# Testcase: NVRAM Partition-IPMI Reprovision
# 1. Update NVRAM config data with test config data
# i.e "nvram --update-config test-name=test-value"
# 2. Issue an IPMI PNOR Reprovision request command, to reset NVRAM partition to default.
# 3. Wait for PNOR Reprovision progress to complete(00).
# 4. Do a Hard reboot(Power OFF/ON) to avoid nvram cache data.
# 5. Once system booted, check for NVRAM parition whether the test config data
# got erased or not.
#
# @return BMC_CONST.FW_SUCCESS or raise OpTestError
#
def test_nvram_ipmi_reprovision(self):
self.cv_SYSTEM.sys_bmc_power_on_validate_host()
self.cv_HOST.host_run_command("uname -a")
self.cv_HOST.host_run_command(BMC_CONST.NVRAM_PRINT_CFG)
print "IPMI_Reprovision: Updating the nvram partition with test cfg data"
self.cv_HOST.host_run_command(BMC_CONST.NVRAM_UPDATE_CONFIG_TEST_DATA)
self.cv_HOST.host_run_command(BMC_CONST.NVRAM_PRINT_CFG)
print "IPMI_Reprovision: issuing ipmi pnor reprovision request"
self.cv_SYSTEM.sys_issue_ipmi_pnor_reprovision_request()
print "IPMI_Reprovision: wait for reprovision to complete"
self.cv_SYSTEM.sys_wait_for_ipmi_pnor_reprovision_to_complete()
print "IPMI_Reprovision: gathering the opal message logs"
self.cv_HOST.host_gather_opal_msg_log()
print "IPMI_Reprovision: Performing a IPMI Power OFF Operation"
# Perform a IPMI Power OFF Operation(Immediate Shutdown)
self.cv_IPMI.ipmi_power_off()
if int(self.cv_SYSTEM.sys_wait_for_standby_state(BMC_CONST.SYSTEM_STANDBY_STATE_DELAY)) == BMC_CONST.FW_SUCCESS:
print "IPMI_Reprovision: System is in standby/Soft-off state"
else:
l_msg = "IPMI_Reprovision: System failed to reach standby/Soft-off state after pnor reprovisioning"
raise OpTestError(l_msg)
print "IPMI_Reprovision: Performing a IPMI Power ON Operation"
# Perform a IPMI Power ON Operation
self.cv_IPMI.ipmi_power_on()
self.cv_SYSTEM.sys_check_host_status()
self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE)
l_res = self.cv_HOST.host_run_command(BMC_CONST.NVRAM_PRINT_CFG)
if l_res.__contains__(BMC_CONST.NVRAM_TEST_DATA):
l_msg = "NVRAM Partition - IPMI Reprovision not happening, nvram test config data still exists"
raise OpTestError(l_msg)
print "NVRAM Partition - IPMI Reprovision is done, cleared the nvram test config data"
self.cv_HOST.host_gather_opal_msg_log()
return BMC_CONST.FW_SUCCESS
##
# @brief This function will cover following test steps
# Testcase: GARD Partition-IPMI Reprovision
# 1. Inject core checkstop using existed function from OpTestHMIHandling.py
# 2. Do a Hard reboot(IPMI Power OFF/ON)
# 2. Issue an IPMI PNOR Reprovision request command, to reset GUARD partition to default.
# 3. Wait for IPMI PNOR Reprovision progress to complete(00).
# 4. Check for GUARD parition whether the existing gard records erased or not.
# 6. Reboot the system back to see system is booting fine or not.
#
# @return BMC_CONST.FW_SUCCESS or raise OpTestError
#
def test_gard_ipmi_reprovision(self):
self.cv_SYSTEM.sys_bmc_power_on_validate_host()
print "IPMI_Reprovision: Injecting system core checkstop to guard the phyisical cpu"
self.opTestHMIHandling.testHMIHandling(BMC_CONST.HMI_MALFUNCTION_ALERT)
print "IPMI_Reprovision: Performing a IPMI Power OFF Operation"
# Perform a IPMI Power OFF Operation(Immediate Shutdown)
self.cv_IPMI.ipmi_power_off()
if int(self.cv_SYSTEM.sys_wait_for_standby_state(BMC_CONST.SYSTEM_STANDBY_STATE_DELAY)) == BMC_CONST.FW_SUCCESS:
print "IPMI_Reprovision: System is in standby/Soft-off state"
else:
l_msg = "IPMI_Reprovision: System failed to reach standby/Soft-off state"
raise OpTestError(l_msg)
print "IPMI_Reprovision: Performing a IPMI Power ON Operation"
# Perform a IPMI Power ON Operation
self.cv_IPMI.ipmi_power_on()
self.cv_SYSTEM.sys_check_host_status()
self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE)
print "IPMI_Reprovision: issuing ipmi pnor reprovision request"
self.cv_SYSTEM.sys_issue_ipmi_pnor_reprovision_request()
print "IPMI_Reprovision: wait for reprovision to complete"
self.cv_SYSTEM.sys_wait_for_ipmi_pnor_reprovision_to_complete()
print "IPMI_Reprovision: gathering the opal message logs"
self.cv_HOST.host_gather_opal_msg_log()
self.cv_HOST.host_get_OS_Level()
self.cv_HOST.host_clone_skiboot_source(BMC_CONST.CLONE_SKIBOOT_DIR)
# Compile the necessary tools xscom-utils and gard utility
self.cv_HOST.host_compile_gard_utility(BMC_CONST.CLONE_SKIBOOT_DIR)
l_con = self.cv_SYSTEM.sys_get_ipmi_console()
self.cv_IPMI.ipmi_host_login(l_con)
self.cv_IPMI.ipmi_host_set_unique_prompt(l_con)
self.cv_IPMI.run_host_cmd_on_ipmi_console("uname -a")
l_res = self.cv_SYSTEM.sys_list_gard_records(BMC_CONST.GARD_TOOL_DIR)
print l_res
if BMC_CONST.NO_GARD_RECORDS in l_res:
print "GUARD Partition - IPMI Reprovision is done, cleared HW gard records"
else:
l_msg = "IPMI: Reprovision not happening, gard records are not erased"
raise OpTestError(l_msg)
print "IPMI_Reprovision: Performing a IPMI Power OFF Operation"
# Perform a IPMI Power OFF Operation(Immediate Shutdown)
self.cv_IPMI.ipmi_power_off()
if int(self.cv_SYSTEM.sys_wait_for_standby_state(BMC_CONST.SYSTEM_STANDBY_STATE_DELAY)) == BMC_CONST.FW_SUCCESS:
print "IPMI_Reprovision: System is in standby/Soft-off state"
else:
l_msg = "IPMI_Reprovision: System failed to reach standby/Soft-off state"
raise OpTestError(l_msg)
print "IPMI_Reprovision: Performing a IPMI Power ON Operation"
# Perform a IPMI Power ON Operation
self.cv_IPMI.ipmi_power_on()
self.cv_SYSTEM.sys_check_host_status()
self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE)
print "IPMI_Reprovision: gathering the opal message logs"
self.cv_HOST.host_gather_opal_msg_log()
return BMC_CONST.FW_SUCCESS
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_BMC = conf.bmc()
self.cv_SYSTEM = conf.system()
self.util = OpTestUtil()
self.prompt = self.util.build_prompt()
class OpTestIPMIPowerControl():
## Initialize this object
# @param i_bmcIP The IP address of the BMC
# @param i_bmcUser The userid to log into the BMC with
# @param i_bmcPasswd The password of the userid to log into the BMC with
# @param i_bmcUserIpmi The userid to issue the BMC IPMI commands with
# @param i_bmcPasswdIpmi The password of BMC IPMI userid
# @param i_ffdcDir Optional param to indicate where to write FFDC
#
# "Only required for inband tests" else Default = None
# @param i_hostIP The IP address of the HOST
# @param i_hostuser The userid to log into the HOST
# @param i_hostPasswd The password of the userid to log into the HOST with
#
def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd,
i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_hostip=None,
i_hostuser=None, i_hostPasswd=None):
self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir)
self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi,
i_ffdcDir, i_hostip, i_hostuser, i_hostPasswd)
self.cv_HOST = OpTestHost(i_hostip, i_hostuser, i_hostPasswd, i_bmcIP)
self.cv_SYSTEM = OpTestSystem(i_bmcIP, i_bmcUser, i_bmcPasswd,
i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostip,
i_hostuser, i_hostPasswd)
self.util = OpTestUtil()
##
# @brief This function will test below system power control operations
# IPMI Power ON
# Power OFF
# Power Soft
# Power Cycle
# Power Reset
# So each operation is executed through ipmi commands. and
# check_system_status function will check whether FW and Host OS
# Boot completed or not.
#
# @return BMC_CONST.FW_SUCCESS or raise OpTestError
#
def testIPMIPowerControl(self):
self.cv_SYSTEM.sys_bmc_power_on_validate_host()
print "Performing a IPMI Power OFF Operation"
# Perform a IPMI Power OFF Operation(Immediate Shutdown)
self.cv_IPMI.ipmi_power_off()
if int(self.cv_SYSTEM.sys_wait_for_standby_state(BMC_CONST.SYSTEM_STANDBY_STATE_DELAY)) == 0:
print "System is in standby/Soft-off state"
else:
l_msg = "System failed to reach standby/Soft-off state"
raise OpTestError(l_msg)
print "Performing a IPMI Power ON Operation"
# Perform a IPMI Power ON Operation
self.cv_IPMI.ipmi_power_on()
self.check_system_status()
self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE)
print "Performing a IPMI Soft Power OFF Operation"
# Perform a IPMI Soft Power OFF Operation(Graceful shutdown)
self.cv_IPMI.ipmi_power_soft()
if int(self.cv_SYSTEM.sys_wait_for_standby_state(BMC_CONST.SYSTEM_STANDBY_STATE_DELAY)) == 0:
print "System is in standby/Soft-off state"
else:
l_msg = "System failed to reach standby/Soft-off state"
raise OpTestError(l_msg)
print "Perform a IPMI Power ON Operation"
# Perform a IPMI Power ON Operation
self.cv_IPMI.ipmi_power_on()
self.check_system_status()
self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE)
print "Performing a IPMI Power Cycle(Soft reboot) Operation "
# Perform a IPMI Power Cycle(Soft reboot) Operation only when system is in ON state
self.cv_IPMI.ipmi_power_cycle()
self.check_system_status()
self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE)
print "Performing a IPMI Power Hard Reset Operation"
# Perform a IPMI Power Hard Reset Operation
self.cv_IPMI.ipmi_power_reset()
self.check_system_status()
self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE)
return BMC_CONST.FW_SUCCESS
##
# @brief This function will check for system status and wait for
# FW and Host OS Boot progress to complete.
#
# @return BMC_CONST.FW_SUCCESS or raise OpTestError
#
def check_system_status(self):
if int(self.cv_SYSTEM.sys_ipl_wait_for_working_state()) == 0:
print "System booted to working state"
else:
l_msg = "System failed to boot"
raise OpTestError(l_msg)
if int(self.cv_SYSTEM.sys_wait_for_os_boot_complete()) == 0:
print "System booted to Host OS"
else:
l_msg = "System failed to boot Host OS"
raise OpTestError(l_msg)
return BMC_CONST.FW_SUCCESS
class HMCUtil():
'''
Utility and functions of HMC object
'''
def __init__(self,
hmc_ip,
user_name,
password,
scratch_disk="",
proxy="",
logfile=sys.stdout,
managed_system=None,
lpar_name=None,
prompt=None,
block_setup_term=None,
delaybeforesend=None,
timeout_factor=None,
lpar_prof=None,
lpar_vios=None,
lpar_user=None,
lpar_password=None,
check_ssh_keys=False,
known_hosts_file=None,
tgt_managed_system=None):
self.hmc_ip = hmc_ip
self.user = user_name
self.passwd = password
self.logfile = logfile
self.mg_system = managed_system
self.tgt_mg_system = tgt_managed_system
self.check_ssh_keys = check_ssh_keys
self.known_hosts_file = known_hosts_file
self.lpar_name = lpar_name
self.lpar_prof = lpar_prof
self.lpar_user = lpar_user
self.lpar_password = lpar_password
self.lpar_vios = lpar_vios
self.util = OpTestUtil()
self.prompt = prompt
self.expect_prompt = self.util.build_prompt(prompt) + "$"
self.ssh = OpTestSSH(hmc_ip,
user_name,
password,
logfile=self.logfile,
check_ssh_keys=check_ssh_keys,
known_hosts_file=known_hosts_file,
block_setup_term=block_setup_term)
self.scratch_disk = scratch_disk
self.proxy = proxy
self.scratch_disk_size = None
self.delaybeforesend = delaybeforesend
self.system = None
# OpTestUtil instance is NOT conf's
self.pty = None
# allows caller specific control of when to block setup_term
self.block_setup_term = block_setup_term
# tells setup_term to not throw exceptions, like when system off
self.setup_term_quiet = 0
# flags the object to abandon setup_term operations, like when system off
self.setup_term_disable = 0
# functional simulators are very slow, so multiply all default timeouts by this factor
self.timeout_factor = timeout_factor
# state tracking, reset on boot and state changes
# console tracking done on System object for the system console
self.PS1_set = -1
self.LOGIN_set = -1
self.SUDO_set = -1
def deactivate_lpar_console(self):
self.ssh.run_command("rmvterm -m %s -p %s" %
(self.mg_system, self.lpar_name),
timeout=10)
def poweroff_system(self):
if self.get_system_state() != OpManagedState.OPERATING:
raise OpTestError('Managed Systen not in Operating state')
self.ssh.run_command("chsysstate -m %s -r sys -o off" % self.mg_system)
self.wait_system_state(OpManagedState.OFF)
def poweron_system(self):
if self.get_system_state() != OpManagedState.OFF:
raise OpTestError('Managed Systen not is Power off state!')
self.ssh.run_command("chsysstate -m %s -r sys -o on" % self.mg_system)
self.wait_system_state()
if self.lpar_vios:
log.debug("Starting VIOS %s", self.lpar_vios)
self.poweron_lpar(vios=True)
def poweroff_lpar(self):
if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
log.info('LPAR Already powered-off!')
return
self.ssh.run_command(
"chsysstate -m %s -r lpar -n %s -o shutdown --immed" %
(self.mg_system, self.lpar_name))
self.wait_lpar_state(OpHmcState.NOT_ACTIVE)
def poweron_lpar(self, vios=False):
if self.get_lpar_state(vios) == OpHmcState.RUNNING:
log.info('LPAR Already powered on!')
return BMC_CONST.FW_SUCCESS
lpar_name = self.lpar_name
if vios:
lpar_name = self.lpar_vios
cmd = "chsysstate -m %s -r lpar -n %s -o on" % (self.mg_system,
lpar_name)
if self.lpar_prof:
cmd = "%s -f %s" % (cmd, self.lpar_prof)
self.wait_lpar_state(OpHmcState.NOT_ACTIVE, vios=vios)
self.ssh.run_command(cmd)
self.wait_lpar_state(vios=vios)
time.sleep(STALLTIME)
return BMC_CONST.FW_SUCCESS
def dumprestart_lpar(self):
if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
log.info('LPAR Already powered-off!')
return
self.ssh.run_command("chsysstate -m %s -r lpar -n %s -o dumprestart" %
(self.mg_system, self.lpar_name))
self.wait_lpar_state()
def restart_lpar(self):
if self.get_lpar_state() in [OpHmcState.NOT_ACTIVE, OpHmcState.NA]:
log.info('LPAR Already powered-off!')
return
self.ssh.run_command(
"chsysstate -m %s -r lpar -n %s -o shutdown --immed --restart" %
(self.mg_system, self.lpar_name))
self.wait_lpar_state()
def get_lpar_state(self, vios=False):
lpar_name = self.lpar_name
if vios:
lpar_name = self.lpar_vios
state = self.ssh.run_command(
'lssyscfg -m %s -r lpar --filter lpar_names=%s -F state' %
(self.mg_system, lpar_name))[-1]
ref_code = self.ssh.run_command(
'lsrefcode -m %s -r lpar --filter lpar_names=%s -F refcode' %
(self.mg_system, lpar_name))[-1]
if state == 'Running':
if 'Linux' in ref_code or not ref_code:
return 'Running'
else:
return 'Booting'
return state
def get_system_state(self):
state = self.ssh.run_command('lssyscfg -m %s -r sys -F state' %
self.mg_system)
return state[-1]
def wait_lpar_state(self,
exp_state=OpHmcState.RUNNING,
vios=False,
timeout=WAITTIME):
state = self.get_lpar_state(vios)
count = 0
while state != exp_state:
state = self.get_lpar_state(vios)
log.info("Current state: %s", state)
time.sleep(timeout)
count = 1
if count > 120:
raise OpTestError("Time exceeded for reaching %s" % exp_state)
def wait_system_state(self,
exp_state=OpManagedState.OPERATING,
timeout=WAITTIME):
state = self.get_system_state()
count = 0
while state != exp_state:
state = self.get_system_state()
log.info("Current state: %s", state)
time.sleep(timeout)
count = 1
if count > 60:
raise OpTestError("Time exceeded for reaching %s" % exp_state)
def is_lpar_in_managed_system(self, mg_system=None):
lpar_list = self.ssh.run_command('lssyscfg -r lpar -m %s -F name' %
mg_system)
if self.lpar_name in lpar_list:
log.info("%s lpar found in managed system %s" %
(mg_system, self.lpar_name))
return True
return False
def migrate_lpar(self, src_mg_system=None, dest_mg_system=None):
if src_mg_system == None or dest_mg_system == None:
raise OpTestError(
"Source and Destination Managed System required for LPM")
if not self.is_lpar_in_managed_system(src_mg_system):
raise OpTestError("Lpar %s not found in managed system %s" %
(self.lpar_name, src_mg_system))
self.ssh.run_command('migrlpar -o v -m %s -t %s -p %s' %
(src_mg_system, dest_mg_system, self.lpar_name))
self.ssh.run_command('migrlpar -o m -m %s -t %s -p %s' %
(src_mg_system, dest_mg_system, self.lpar_name))
if self.is_lpar_in_managed_system(dest_mg_system):
log.info("Migration of lpar %s from %s to %s is successfull" %
(self.lpar_name, src_mg_system, dest_mg_system))
self.mg_system = dest_mg_system
return True
log.info("Migration of lpar %s from %s to %s failed" %
(self.lpar_name, src_mg_system, dest_mg_system))
return False
def run_command_ignore_fail(self, command, timeout=60, retry=0):
return self.ssh.run_command_ignore_fail(command,
timeout * self.timeout_factor,
retry)
def run_command(self, i_cmd, timeout=15):
return self.ssh.run_command(i_cmd, timeout)
class OpTestFWTS():
## Initialize this object
# @param i_bmcIP The IP address of the BMC
# @param i_bmcUser The userid to log into the BMC with
# @param i_bmcPasswd The password of the userid to log into the BMC with
# @param i_bmcUserIpmi The userid to issue the BMC IPMI commands with
# @param i_bmcPasswdIpmi The password of BMC IPMI userid
# @param i_ffdcDir Optional param to indicate where to write FFDC
#
# "Only required for inband tests" else Default = None
# @param i_hostIP The IP address of the HOST
# @param i_hostuser The userid to log into the HOST
# @param i_hostPasswd The password of the userid to log into the HOST with
#
def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd,
i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_hostip=None,
i_hostuser=None, i_hostPasswd=None):
self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir)
self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi,
i_ffdcDir, i_hostip, i_hostuser, i_hostPasswd)
self.cv_HOST = OpTestHost(i_hostip, i_hostuser, i_hostPasswd, i_bmcIP, i_ffdcDir)
self.cv_SYSTEM = OpTestSystem(i_bmcIP, i_bmcUser, i_bmcPasswd,
i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir, i_hostip,
i_hostuser, i_hostPasswd)
self.util = OpTestUtil()
self.user = i_hostuser
self.ip = i_hostip
self.passwd = i_hostPasswd
##
# @brief This function just brings the system to host OS.
#
# @return BMC_CONST.FW_SUCCESS or raise OpTestError
#
def test_system_reboot(self):
print "Testing FWTS: Booting system to OS"
print "Performing a IPMI Power OFF Operation"
# Perform a IPMI Power OFF Operation(Immediate Shutdown)
self.cv_IPMI.ipmi_power_off()
if int(self.cv_SYSTEM.sys_wait_for_standby_state(BMC_CONST.SYSTEM_STANDBY_STATE_DELAY)) == BMC_CONST.FW_SUCCESS:
print "System is in standby/Soft-off state"
else:
l_msg = "System failed to reach standby/Soft-off state"
raise OpTestError(l_msg)
self.cv_IPMI.ipmi_power_on()
self.cv_SYSTEM.sys_check_host_status()
self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE)
self.cv_IPMI.clear_ssh_keys(self.cv_HOST.ip)
print "Gathering the OPAL msg logs"
self.cv_HOST.host_gather_opal_msg_log()
return BMC_CONST.FW_SUCCESS
##
# @brief This function just executes the fwts_execution.sh on host OS
#
# @return BMC_CONST.FW_SUCCESS or raise OpTestError
#
def test_fwts(self):
l_oslevel = self.cv_HOST.host_get_OS_Level()
if not "Ubuntu" in l_oslevel:
return
# Copy the fwts execution file to the tmp folder in the host
base_path = (os.path.dirname(os.path.abspath(__file__))).split('testcases')[0]
fwts_script = base_path + "/testcases/fwts_execution.sh"
try:
self.util.copyFilesToDest(fwts_script, self.user,
self.ip, "/tmp/", self.passwd)
except:
l_msg = "Copying fwts file to host failed"
print l_msg
raise OpTestError(l_msg)
l_res = self.cv_HOST.host_run_command("/tmp/fwts_execution.sh")
print l_res
class OpTestMtdPnorDriver():
## Initialize this object and also getting the host login credentials to use by scp utility
# @param i_bmcIP The IP address of the BMC
# @param i_bmcUser The userid to log into the BMC with
# @param i_bmcPasswd The password of the userid to log into the BMC with
# @param i_bmcUserIpmi The userid to issue the BMC IPMI commands with
# @param i_bmcPasswdIpmi The password of BMC IPMI userid
# @param i_ffdcDir Optional param to indicate where to write FFDC
#
# "Only required for inband tests" else Default = None
# @param i_hostIP The IP address of the HOST
# @param i_hostuser The userid to log into the HOST
# @param i_hostPasswd The password of the userid to log into the HOST with
#
def __init__(self,
i_bmcIP,
i_bmcUser,
i_bmcPasswd,
i_bmcUserIpmi,
i_bmcPasswdIpmi,
i_ffdcDir=None,
i_hostip=None,
i_hostuser=None,
i_hostPasswd=None):
self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir)
self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi,
i_ffdcDir)
self.cv_HOST = OpTestHost(i_hostip, i_hostuser, i_hostPasswd, i_bmcIP)
self.cv_SYSTEM = OpTestSystem(i_bmcIP, i_bmcUser, i_bmcPasswd,
i_bmcUserIpmi, i_bmcPasswdIpmi,
i_ffdcDir, i_hostip, i_hostuser,
i_hostPasswd)
self.util = OpTestUtil()
self.host_user = i_hostuser
self.host_ip = i_hostip
self.host_Passwd = i_hostPasswd
##
# @brief This function has following test steps
# 1. Get host information(OS and Kernel information)
# 2. Load the mtd module based on config value
# 3. Check /dev/mtd0 character device file existence on host
# 4. Copying the contents of the flash in a file /tmp/pnor
# 5. Getting the /tmp/pnor file into local x86 machine using scp utility
# 6. Remove existing /tmp/ffs directory and
# Clone latest ffs git repository in local x86 working machine
# 7. Compile ffs repository to get fcp utility
# 8. Check existence of fcp utility in ffs repository, after compiling
# 9. Get the PNOR flash contents on an x86 machine using fcp utility
#
# @return BMC_CONST.FW_SUCCESS-success or raise OpTestError-fail
#
def testMtdPnorDriver(self):
self.cv_SYSTEM.sys_bmc_power_on_validate_host()
# Get OS level
l_oslevel = self.cv_HOST.host_get_OS_Level()
# Get Kernel Version
l_kernel = self.cv_HOST.host_get_kernel_version()
# loading mtd module based on config option
l_config = "CONFIG_MTD_POWERNV_FLASH"
l_module = "mtd"
self.cv_HOST.host_load_module_based_on_config(l_kernel, l_config,
l_module)
# Check /dev/mtd0 file existence on host
l_cmd = "ls -l /dev/mtd0; echo $?"
l_res = self.cv_HOST.host_run_command(l_cmd)
l_res = l_res.splitlines()
if int(l_res[-1]) == 0:
print "/dev/mtd0 character device file exists on host"
else:
l_msg = "/dev/mtd0 character device file doesn't exist on host"
print l_msg
raise OpTestError(l_msg)
# Copying the contents of the PNOR flash in a file /tmp/pnor
l_file = "/tmp/pnor"
l_cmd = "cat /dev/mtd0 > %s; echo $?" % l_file
l_res = self.cv_HOST.host_run_command(l_cmd)
l_res = l_res.splitlines()
if int(l_res[-1]) == 0:
print "Fetched PNOR data from /dev/mtd0 into temp file /tmp/pnor"
else:
l_msg = "Fetching PNOR data is failed from /dev/mtd0 into temp file /tmp/pnor"
print l_msg
raise OpTestError(l_msg)
# Getting the /tmp/pnor file into local x86 machine
l_path = "/tmp/"
self.util.copyFilesToDest(l_path, self.host_user, self.host_ip, l_file,
self.host_Passwd, "2",
BMC_CONST.SCP_TO_LOCAL)
l_list = commands.getstatusoutput("ls -l %s; echo $?" % l_path)
print l_list
l_workdir = "/tmp/ffs"
# Remove existing /tmp/ffs directory
l_res = commands.getstatusoutput("rm -rf %s" % l_workdir)
print l_res
# Clone latest ffs git repository in local x86 working machine
l_cmd = "git clone https://github.com/open-power/ffs/ %s" % l_workdir
l_res = commands.getstatusoutput(l_cmd)
print l_res
if int(l_res[0]) == 0:
print "Cloning of ffs repository is successfull"
else:
l_msg = "Cloning ffs repository is failed"
print l_msg
raise OpTestError(l_msg)
# Compile ffs repository to get fcp utility
l_cmd = "cd %s/; make" % l_workdir
l_res = commands.getstatusoutput(l_cmd)
print l_res
if int(l_res[0]) == 0:
print "Compiling fcp utility is successfull"
else:
l_msg = "Compiling fcp utility is failed"
print l_msg
raise OpTestError(l_msg)
# Check existence of fcp utility in ffs repository, after compiling.
l_cmd = "test -f %s/fcp/x86/fcp" % l_workdir
l_res = commands.getstatusoutput(l_cmd)
print l_res
if int(l_res[0]) == 0:
print "Compiling fcp utility is successfull"
else:
l_msg = "Compiling fcp utility is failed"
print l_msg
raise OpTestError(l_msg)
# Check the PNOR flash contents on an x86 machine using fcp utility
l_cmd = "%s/fcp/x86/fcp -o 0x0 -L %s" % (l_workdir, l_file)
l_res = commands.getstatusoutput(l_cmd)
print l_res[1]
if int(l_res[0]) == 0:
print "Getting PNOR data successfull using fcp utility"
return BMC_CONST.FW_SUCCESS
else:
l_msg = "Getting the PNOR data using fcp utility failed"
print l_msg
raise OpTestError(l_msg)
class HMCConsole(HMCUtil):
"""
HMCConsole Class
Methods to manage the console of LPAR
"""
def __init__(self,
hmc_ip,
user_name,
password,
managed_system,
lpar_name,
lpar_vios,
lpar_prof,
lpar_user,
lpar_password,
block_setup_term=None,
delaybeforesend=None,
timeout_factor=1,
logfile=sys.stdout,
prompt=None,
scratch_disk="",
check_ssh_keys=False,
known_hosts_file=None,
proxy=""):
self.logfile = logfile
self.hmc_ip = hmc_ip
self.user = user_name
self.passwd = password
self.mg_system = managed_system
self.util = OpTestUtil()
self.expect_prompt = self.util.build_prompt(prompt) + "$"
self.lpar_name = lpar_name
self.lpar_vios = lpar_vios
self.lpar_prof = lpar_prof
self.lpar_user = lpar_user
self.lpar_password = lpar_password
self.scratch_disk = scratch_disk
self.proxy = proxy
self.state = ConsoleState.DISCONNECTED
self.delaybeforesend = delaybeforesend
self.system = None
# OpTestUtil instance is NOT conf's
self.prompt = prompt
self.pty = None
self.delaybeforesend = delaybeforesend
# allows caller specific control of when to block setup_term
self.block_setup_term = block_setup_term
# tells setup_term to not throw exceptions, like when system off
self.setup_term_quiet = 0
# flags the object to abandon setup_term operations, like when system off
self.setup_term_disable = 0
# FUTURE - System Console currently tracked in System Object
# state tracking, reset on boot and state changes
self.PS1_set = -1
self.LOGIN_set = -1
self.SUDO_set = -1
super(HMCConsole,
self).__init__(hmc_ip, user_name, password, scratch_disk, proxy,
logfile, managed_system, lpar_name, prompt,
block_setup_term, delaybeforesend, timeout_factor,
lpar_prof, lpar_vios, lpar_user, lpar_password,
check_ssh_keys, known_hosts_file)
def set_system(self, system):
self.ssh.set_system(system)
self.system = system
self.pty = self.get_console()
self.pty.set_system(system)
def get_host_console(self):
return self.pty
def set_system_setup_term(self, flag):
self.system.block_setup_term = flag
def get_system_setup_term(self):
return self.system.block_setup_term
def get_scratch_disk(self):
return self.scratch_disk
def get_proxy(self):
return self.proxy
def hostname(self):
return self.hmc_ip
def username(self):
return self.user
def password(self):
return self.passwd
def set_block_setup_term(self, flag):
self.block_setup_term = flag
def get_block_setup_term(self):
return self.block_setup_term
def enable_setup_term_quiet(self):
self.setup_term_quiet = 1
self.setup_term_disable = 0
def disable_setup_term_quiet(self):
self.setup_term_quiet = 0
self.setup_term_disable = 0
def close(self):
self.util.clear_state(self)
try:
self.pty.close()
if self.pty.status != -1: # leaving for debug
if os.WIFEXITED(self.pty.status):
os.WEXITSTATUS(self.pty.status)
else:
os.WTERMSIG(self.pty.status)
self.state = ConsoleState.DISCONNECTED
except pexpect.ExceptionPexpect:
self.state = ConsoleState.DISCONNECTED
raise "HMC Console: failed to close console"
except Exception:
self.state = ConsoleState.DISCONNECTED
log.debug("HMC close -> TERMINATE")
def connect(self, logger=None):
if self.state == ConsoleState.CONNECTED:
return self.pty
self.util.clear_state(self) # clear when coming in DISCONNECTED
log.info("De-activating the console")
self.deactivate_lpar_console()
log.debug("#HMC Console CONNECT")
command = "sshpass -p %s ssh -p 22 -l %s %s -o PubkeyAuthentication=no"\
" -o afstokenpassing=no -q -o 'UserKnownHostsFile=/dev/null'"\
" -o 'StrictHostKeyChecking=no'"
try:
self.pty = Spawn(command % (self.passwd, self.user, self.hmc_ip))
log.info("Opening the LPAR console")
time.sleep(STALLTIME)
self.pty.send('\r')
self.pty.sendline("mkvterm -m %s -p %s" %
(self.mg_system, self.lpar_name))
self.pty.send('\r')
time.sleep(STALLTIME)
i = self.pty.expect(["Open Completed.", pexpect.TIMEOUT],
timeout=60)
self.pty.logfile = sys.stdout
if logger:
self.pty.logfile_read = OpTestLogger.FileLikeLogger(logger)
else:
self.pty.logfile_read = OpTestLogger.FileLikeLogger(log)
if i == 0:
time.sleep(STALLTIME)
self.state = ConsoleState.CONNECTED
self.pty.setwinsize(1000, 1000)
else:
raise OpTestError("Check the lpar activate command")
except Exception as exp:
self.state = ConsoleState.DISCONNECTED
raise CommandFailed(
'OPexpect.spawn',
'OPexpect.spawn encountered a problem: ' + str(exp), -1)
if self.delaybeforesend:
self.pty.delaybeforesend = self.delaybeforesend
if not self.pty.isalive():
raise CommandFailed("mkvterm", self.pty.read(), self.pty.status)
return self.pty
def check_state(self):
return self.state
def get_console(self, logger=None):
if self.state == ConsoleState.DISCONNECTED:
self.util.clear_state(self)
self.connect(logger=logger)
time.sleep(STALLTIME)
l_rc = self.pty.expect(["login:"******"Shell prompt changed")
else:
self.pty.send('\r')
log.debug("Waiting till booting!")
self.pty = self.get_login_prompt()
if self.system.SUDO_set != 1 or self.system.LOGIN_set != 1 or self.system.PS1_set != 1:
self.util.setup_term(self.system, self.pty, None,
self.system.block_setup_term)
# Clear buffer before usage
self.pty.buffer = ""
return self.pty
def get_login_prompt(self):
# Assuming 'Normal' boot set in LPAR profile
# We wait for upto 500 seconds for LPAR to boot to OS
self.pty.send('\r')
time.sleep(STALLTIME)
log.debug("Waiting for login screen")
i = self.pty.expect(["login:"******"System has booted")
time.sleep(STALLTIME)
elif i == 1:
log.debug("Console already logged in")
else:
log.error("Failed to get login prompt %s", self.pty.before)
# To cheat system for making using of HMC SSH
self.system.PS1_set = 1
self.system.LOGIN_set = 1
self.system.SUDO_set = 1
return self.pty
def run_command(self, i_cmd, timeout=15):
return self.util.run_command(self, i_cmd, timeout)
class OpTestFlashBase(unittest.TestCase):
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_SYSTEM = conf.system()
self.cv_BMC = conf.bmc()
self.cv_REST = self.cv_BMC.get_rest_api()
self.cv_HOST = conf.host()
self.cv_IPMI = conf.ipmi()
self.platform = conf.platform()
self.util = OpTestUtil()
self.bmc_type = conf.args.bmc_type
self.bmc_ip = conf.args.bmc_ip
self.bmc_username = conf.args.bmc_username
self.bmc_password = conf.args.bmc_password
self.pupdate_binary = conf.args.pupdate
self.pflash = conf.args.pflash
def validate_side_activated(self):
l_bmc_side, l_pnor_side = self.cv_IPMI.ipmi_get_side_activated()
self.assertIn(BMC_CONST.PRIMARY_SIDE, l_bmc_side,
"BMC: Primary side is not active")
if (l_pnor_side == BMC_CONST.GOLDEN_SIDE):
print "PNOR: Primary side is not active"
bios_sensor = self.cv_IPMI.ipmi_get_golden_side_sensor_id()
self.assertNotEqual(
bios_sensor, None,
"Failed to get the BIOS Golden side sensor id")
boot_count_sensor = self.cv_IPMI.ipmi_get_boot_count_sensor_id()
self.assertNotEqual(boot_count_sensor, None,
"Failed to get the Boot Count sensor id")
self.cv_IPMI.ipmi_set_pnor_primary_side(bios_sensor,
boot_count_sensor)
l_bmc_side, l_pnor_side = self.cv_IPMI.ipmi_get_side_activated()
self.assertIn(BMC_CONST.PRIMARY_SIDE, l_pnor_side,
"PNOR: Primary side is not active")
def get_pnor_level(self):
rc = self.cv_IPMI.ipmi_get_PNOR_level()
print rc
def bmc_down_check(self):
self.assertTrue(self.util.ping_fail_check(self.cv_BMC.host_name),
"FSP/BMC keeps on pinging up")
def scp_file(self, src_file_path, dst_file_path):
self.util.copyFilesToDest(src_file_path, self.bmc_username,
self.bmc_ip, dst_file_path,
self.bmc_password, "2",
BMC_CONST.SCP_TO_REMOTE)
def get_version_tar(self, file_path):
tar = tarfile.open(file_path)
for member in tar.getmembers():
fd = tar.extractfile(member)
content = fd.read()
if "version=" in content:
content = content.split("\n")
content = [x for x in content if "version=" in x]
version = content[0].split("=")[-1]
break
tar.close()
print version
return version
def get_image_version(self, path):
output = self.cv_BMC.run_command("cat %s | grep \"version=\"" % path)
return output[0].split("=")[-1]
def delete_images_dir(self):
try:
self.cv_BMC.run_command("rm -rf /tmp/images/*")
except CommandFailed:
pass
def get_image_path(self, image_version):
retry = 0
while (retry < 20):
image_list = []
try:
image_list = self.cv_BMC.run_command(
"ls -1 -d /tmp/images/*/ --color=never")
except CommandFailed as cf:
pass
for i in range(0, len(image_list)):
version = self.get_image_version(image_list[i] + "MANIFEST")
if (version == image_version):
return image_list[i]
time.sleep(5)
retry += 1
def get_image_id(self, version):
img_path = self.get_image_path(version)
img_id = img_path.split("/")[-2]
print "Image id for Host image is : %s" % img_id
return img_id
def wait_for_bmc_runtime(self):
self.util.PingFunc(self.bmc_ip, BMC_CONST.PING_RETRY_FOR_STABILITY)
if "SMC" in self.bmc_type:
self.cv_IPMI.ipmi_wait_for_bmc_runtime()
elif "OpenBMC" in self.bmc_type:
self.cv_REST.wait_for_bmc_runtime()
return
class OpTestHMIHandling(unittest.TestCase):
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_HOST = conf.host()
self.cv_IPMI = conf.ipmi()
self.cv_FSP = conf.bmc()
self.cv_SYSTEM = conf.system()
self.bmc_type = conf.args.bmc_type
self.util = OpTestUtil()
def ipmi_monitor_sol_ipl(self, console, timeout):
# Error injection causing the SOL console to terminate immediately.
# So Let's re-connect the console
console.close()
self.cv_SYSTEM.set_state(OpSystemState.IPLing)
try:
self.cv_SYSTEM.goto_state(OpSystemState.OS)
except:
self.util.PingFunc(self.cv_HOST.ip,
BMC_CONST.PING_RETRY_POWERCYCLE)
self.cv_SYSTEM.set_state(OpSystemState.OS)
console.close()
print "System booted fine to host OS..."
return BMC_CONST.FW_SUCCESS
def verify_proc_recovery(self, l_res):
if any("Processor Recovery done" in line for line in l_res) and \
any("Harmless Hypervisor Maintenance interrupt [Recovered]" in line for line in l_res):
print "Processor recovery done"
return
else:
raise Exception(
"HMI handling failed to log message: for proc_recv_done")
def verify_timer_facility_recovery(self, l_res):
if any("Timer facility experienced an error" in line for line in l_res) and \
any("Severe Hypervisor Maintenance interrupt [Recovered]" in line for line in l_res):
print "Timer facility experienced an error and got recovered"
return
else:
raise Exception("HMI handling failed to log message")
def init_test(self):
self.cv_SYSTEM.goto_state(OpSystemState.OS)
self.proc_gen = self.cv_HOST.host_get_proc_gen()
l_chips = self.cv_HOST.host_get_list_of_chips(
) # ['00000000', '00000001', '00000010']
if not l_chips:
raise Exception("Getscom failed to list processor chip ids")
l_cores = self.cv_HOST.host_get_cores()
if not l_cores:
raise Exception("Failed to get list of core id's")
print l_cores # {0: ['4', '5', '6', 'c', 'd', 'e'], 1: ['4', '5', '6', 'c', 'd', 'e'], 10: ['4', '5', '6', 'c', 'd', 'e']}
# Remove master core where injecting core checkstop leads to IPL expected failures
# after 2 failures system will starts boot in Golden side of PNOR
l_cores[0][1].pop(0)
print l_cores
self.l_dic = []
i = 0
for tup in l_cores:
new_list = [l_chips[i], tup[1]]
self.l_dic.append(new_list)
i += 1
print self.l_dic
# self.l_dic is a list of chip id's, core id's . and is of below format
# [['00000000', ['4', '5', '6', 'c', 'd', 'e']], ['00000001', ['4', '5', '6', 'c', 'd', 'e']], ['00000010', ['4', '5', '6', 'c', 'd', 'e']]]
# In-order to inject HMI errors on cpu's, cpu should be running, so disabling the sleep states 1 and 2 of all CPU's
self.disable_cpu_idle_states()
# Disable kdump to check behaviour of IPL caused due to kernel panic after injection of core/system checkstop
self.disable_kdump_service()
def disable_kdump_service(self):
l_oslevel = self.cv_HOST.host_get_OS_Level()
try:
if "Ubuntu" in l_oslevel:
self.cv_HOST.host_run_command("service kdump-tools stop")
else:
self.cv_HOST.host_run_command("service kdump stop")
except CommandFailed as cf:
if cf.exitcode == 5:
# kdump may not be enabled, so it's not a failure to stop it
pass
def enable_idle_state(self, i_idle):
l_cmd = "for i in /sys/devices/system/cpu/cpu*/cpuidle/state%s/disable; do echo 0 > $i; done" % i_idle
self.cv_HOST.host_run_command(l_cmd)
def disable_idle_state(self, i_idle):
l_cmd = "for i in /sys/devices/system/cpu/cpu*/cpuidle/state%s/disable; do echo 1 > $i; done" % i_idle
self.cv_HOST.host_run_command(l_cmd)
# Disable all CPU idle states except snooze state
def disable_cpu_idle_states(self):
states = self.cv_HOST.host_run_command(
"find /sys/devices/system/cpu/cpu*/cpuidle/state* -type d | cut -d'/' -f8 | sort -u | sed -e 's/^state//'"
)
for state in states:
if state is "0":
try:
self.cv_HOST.host_run_command("cpupower idle-set -e 0")
except CommandFailed:
self.enable_idle_state("0")
continue
try:
self.cv_HOST.host_run_command("cpupower idle-set -d %s" %
state)
except CommandFailed:
self.disable_idle_state(state)
def form_scom_addr(self, addr, core):
if self.proc_gen in ["POWER8", "POWER8E"]:
val = addr[0] + str(core) + addr[2:]
elif self.proc_gen in ["POWER9"]:
val = hex(
eval("0x%s | (((%s & 0x1f) + 0x20) << 24)" %
(addr, int(core, 16))))
print val
return val
def clearGardEntries(self):
self.cv_SYSTEM.goto_state(OpSystemState.OS)
self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE)
if "FSP" in self.bmc_type:
res = self.cv_FSP.fspc.run_command("gard --clr all")
self.assertIn("Success in clearing Gard Data", res,
"Failed to clear GARD entries")
print self.cv_FSP.fspc.run_command("gard --gc cpu")
else:
g = self.cv_HOST.host_run_command(
"PATH=/usr/local/sbin:$PATH opal-gard list all")
if "No GARD entries to display" not in g:
self.cv_HOST.host_run_command(
"PATH=/usr/local/sbin:$PATH opal-gard clear all")
cleared_gard = self.cv_HOST.host_run_command(
"PATH=/usr/local/sbin:$PATH opal-gard list")
self.assertIn("No GARD entries to display", cleared_gard,
"Failed to clear GARD entries")
self.cv_SYSTEM.goto_state(OpSystemState.OFF)
self.cv_SYSTEM.goto_state(OpSystemState.OS)
##
# @brief This function executes HMI test case based on the i_test value, Before test starts
# disabling kdump service to make sure system reboots, after injecting non-recoverable errors.
#
# @param i_test @type int: this is the type of test case want to execute
# BMC_CONST.HMI_PROC_RECV_DONE: Processor recovery done
# BMC_CONST.HMI_PROC_RECV_ERROR_MASKED: proc_recv_error_masked
# BMC_CONST.HMI_MALFUNCTION_ALERT: malfunction_alert
# BMC_CONST.HMI_HYPERVISOR_RESOURCE_ERROR: hypervisor resource error
def _testHMIHandling(self, i_test):
l_test = i_test
self.init_test()
self.cv_SYSTEM.goto_state(OpSystemState.OS)
self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE)
l_con = self.cv_SYSTEM.sys_get_ipmi_console()
self.cv_SYSTEM.host_console_login()
self.cv_SYSTEM.host_console_unique_prompt()
l_con.run_command("uname -a")
l_con.run_command("cat /etc/os-release")
l_con.run_command("lscpu")
l_con.run_command("dmesg -D")
if l_test == BMC_CONST.HMI_PROC_RECV_DONE:
self._test_proc_recv_done()
elif l_test == BMC_CONST.HMI_PROC_RECV_ERROR_MASKED:
self._test_proc_recv_error_masked()
elif l_test == BMC_CONST.HMI_MALFUNCTION_ALERT:
self._test_malfunction_allert()
elif l_test == BMC_CONST.HMI_HYPERVISOR_RESOURCE_ERROR:
self._test_hyp_resource_err()
elif l_test == BMC_CONST.TOD_ERRORS:
# TOD Error recovery works on systems having more than one chip TOD
# Skip this test on single chip systems(as recovery fails on 1S systems)
if len(self.l_dic) == 1:
l_msg = "This is a single chip system, TOD Error recovery won't work"
print l_msg
return BMC_CONST.FW_SUCCESS
elif len(self.l_dic) > 1:
self._test_tod_errors(BMC_CONST.PSS_HAMMING_DISTANCE)
self._test_tod_errors(BMC_CONST.INTERNAL_PATH_OR_PARITY_ERROR)
self._test_tod_errors(BMC_CONST.TOD_DATA_PARITY_ERROR)
self._test_tod_errors(BMC_CONST.TOD_SYNC_CHECK_ERROR)
self._test_tod_errors(BMC_CONST.FSM_STATE_PARITY_ERROR)
self._test_tod_errors(BMC_CONST.MASTER_PATH_CONTROL_REGISTER)
self._test_tod_errors(
BMC_CONST.PORT_0_PRIMARY_CONFIGURATION_REGISTER)
self._test_tod_errors(
BMC_CONST.PORT_1_PRIMARY_CONFIGURATION_REGISTER)
self._test_tod_errors(
BMC_CONST.PORT_0_SECONDARY_CONFIGURATION_REGISTER)
self._test_tod_errors(
BMC_CONST.PORT_1_SECONDARY_CONFIGURATION_REGISTER)
self._test_tod_errors(BMC_CONST.SLAVE_PATH_CONTROL_REGISTER)
self._test_tod_errors(BMC_CONST.INTERNAL_PATH_CONTROL_REGISTER)
self._test_tod_errors(BMC_CONST.PR_SC_MS_SL_CONTROL_REGISTER)
else:
raise Exception("Getting Chip information failed")
elif l_test == BMC_CONST.TFMR_ERRORS:
self._testTFMR_Errors(BMC_CONST.TB_PARITY_ERROR)
self._testTFMR_Errors(BMC_CONST.TFMR_PARITY_ERROR)
self._testTFMR_Errors(BMC_CONST.TFMR_HDEC_PARITY_ERROR)
self._testTFMR_Errors(BMC_CONST.TFMR_DEC_PARITY_ERROR)
self._testTFMR_Errors(BMC_CONST.TFMR_PURR_PARITY_ERROR)
self._testTFMR_Errors(BMC_CONST.TFMR_SPURR_PARITY_ERROR)
else:
raise Exception("Please provide valid test case")
self.cv_HOST.ssh.state = SSHConnectionState.DISCONNECTED
return BMC_CONST.FW_SUCCESS
##
# @brief This function is used to test HMI: processor recovery done
# and also this function injecting error on all the cpus one by one and
# verify whether cpu is recovered or not.
def _test_proc_recv_done(self):
if self.proc_gen in ["POWER9"]:
scom_addr = "20010A40"
elif self.proc_gen in ["POWER8", "POWER8E"]:
scom_addr = "10013100"
else:
return
for l_pair in self.l_dic:
l_chip = l_pair[0]
for l_core in l_pair[1]:
l_reg = self.form_scom_addr(scom_addr, l_core)
l_cmd = "PATH=/usr/local/sbin:$PATH putscom -c %s %s 0000000000100000" % (
l_chip, l_reg)
console = self.cv_SYSTEM.sys_get_ipmi_console()
console.run_command("dmesg -C")
try:
l_res = console.run_command(l_cmd, timeout=20)
except CommandFailed as cf:
l_res = cf.output
if cf.exitcode == 1:
pass
else:
if any("Kernel panic - not syncing" in line
for line in l_res):
raise Exception(
"Processor recovery failed: Kernel got panic")
elif any("Petitboot" in line for line in l_res):
raise Exception(
"System reached petitboot:Processor recovery failed"
)
elif any("ISTEP" in line for line in l_res):
raise Exception(
"System started booting: Processor recovery failed"
)
else:
raise Exception(
"Failed to inject thread hang recoverable error %s",
str(cf))
time.sleep(0.2)
l_res = console.run_command("dmesg")
self.verify_proc_recovery(l_res)
return
##
# @brief This function is used to test HMI: proc_recv_error_masked
# Processor went through recovery for an error which is actually masked for reporting
# this function also injecting the error on all the cpu's one-by-one.
def _test_proc_recv_error_masked(self):
if self.proc_gen in ["POWER8", "POWER8E"]:
scom_addr = "10013100"
else:
return
for l_pair in self.l_dic:
l_chip = l_pair[0]
for l_core in l_pair[1]:
l_reg = self.form_scom_addr(scom_addr, l_core)
l_cmd = "PATH=/usr/local/sbin:$PATH putscom -c %s %s 0000000000080000" % (
l_chip, l_reg)
console = self.cv_SYSTEM.sys_get_ipmi_console()
console.run_command("dmesg -C")
try:
l_res = console.run_command(l_cmd, timeout=20)
except CommandFailed as cf:
l_res = cf.output
if cf.exitcode == 1:
pass
else:
if any("Kernel panic - not syncing" in line
for line in l_res):
raise Exception(
"Processor recovery failed: Kernel got panic")
elif any("Petitboot" in line for line in l_res):
raise Exception(
"System reached petitboot:Processor recovery failed"
)
elif any("ISTEP" in line for line in l_res):
raise Exception(
"System started booting: Processor recovery failed"
)
else:
raise Exception(
"Failed to inject thread hang recoverable error %s",
str(cf))
time.sleep(0.2)
l_res = console.run_command("dmesg")
self.verify_proc_recovery(l_res)
return
##
# @brief This function is used to test hmi malfunction alert:Core checkstop
# A processor core in the system has to be checkstopped (failed recovery).
# Injecting core checkstop on random core of random chip
def _test_malfunction_allert(self):
if self.proc_gen in ["POWER9"]:
scom_addr = "20010A40"
elif self.proc_gen in ["POWER8", "POWER8E"]:
scom_addr = "10013100"
else:
return
# Get random pair of chip vs cores
l_pair = random.choice(self.l_dic)
# Get random chip id
l_chip = l_pair[0]
# Get random core number
l_core = random.choice(l_pair[1])
l_reg = self.form_scom_addr(scom_addr, l_core)
l_cmd = "PATH=/usr/local/sbin:$PATH putscom -c %s %s 1000000000000000" % (
l_chip, l_reg)
# Core checkstop will lead to system IPL, so we will wait for certain time for IPL
# to finish
#l_res = self.cv_SYSTEM.sys_get_ipmi_console().run_command(l_cmd, timeout=600)
console = self.cv_SYSTEM.sys_get_ipmi_console()
console.sol.sendline(l_cmd)
self.ipmi_monitor_sol_ipl(console, timeout=600)
##
# @brief This function is used to test HMI: Hypervisor resource error
# Injecting Hypervisor resource error on random core of random chip
def _test_hyp_resource_err(self):
if self.proc_gen in ["POWER9"]:
scom_addr = "20010A40"
elif self.proc_gen in ["POWER8", "POWER8E"]:
scom_addr = "10013100"
else:
return
# Get random pair of chip vs cores
l_pair = random.choice(self.l_dic)
# Get random chip id
l_chip = l_pair[0]
# Get random core number
l_core = random.choice(l_pair[1])
l_reg = self.form_scom_addr(scom_addr, l_core)
l_cmd = "PATH=/usr/local/sbin:$PATH putscom -c %s %s 0000000000008000" % (
l_chip, l_reg)
console = self.cv_SYSTEM.sys_get_ipmi_console()
console.sol.sendline(l_cmd)
self.ipmi_monitor_sol_ipl(console, timeout=600)
##
# @brief This function tests timer facility related error injections and check
# the corresponding error got recovered. And this process is repeated
# for all the active cores in all the chips.
#
# @param i_error @type string: this is the type of error want to inject
# BMC_CONST.TB_PARITY_ERROR
# BMC_CONST.TFMR_PARITY_ERROR
# BMC_CONST.TFMR_HDEC_PARITY_ERROR
# BMC_CONST.TFMR_DEC_PARITY_ERROR
# BMC_CONST.TFMR_PURR_PARITY_ERROR
# BMC_CONST.TFMR_SPURR_PARITY_ERROR
def _testTFMR_Errors(self, i_error):
if self.proc_gen in ["POWER9"]:
scom_addr = "20010A84"
elif self.proc_gen in ["POWER8", "POWER8E"]:
scom_addr = "10013281"
else:
return
l_error = i_error
for l_pair in self.l_dic:
l_chip = l_pair[0]
for l_core in l_pair[1]:
l_reg = self.form_scom_addr(scom_addr, l_core)
l_cmd = "PATH=/usr/local/sbin:$PATH putscom -c %s %s %s" % (
l_chip, l_reg, l_error)
console = self.cv_SYSTEM.sys_get_ipmi_console()
console.run_command("dmesg -C")
try:
l_res = console.run_command(l_cmd, timeout=20)
except CommandFailed as cf:
l_res = cf.output
if cf.exitcode == 1:
pass
else:
if any("Kernel panic - not syncing" in line
for line in l_res):
l_msg = "TFMR error injection: Kernel got panic"
elif any("Petitboot" in line for line in l_res):
l_msg = "System reached petitboot:TFMR error injection recovery failed"
elif any("ISTEP" in line for line in l_res):
l_msg = "System started booting: TFMR error injection recovery failed"
else:
raise Exception("Failed to inject TFMR error %s " %
str(cf))
time.sleep(0.2)
l_res = console.run_command("dmesg")
self.verify_timer_facility_recovery(l_res)
return
##
# @brief This function tests chip TOD related error injections and check
# the corresponding error got recovered. And this error injection
# happening on a random chip. This tod errors should test on systems
# having more than one processor socket(chip). On single chip system
# TOD error recovery won't work.
#
# @param i_error @type string: this is the type of error want to inject
# These errors represented in common/OpTestConstants.py file.
def _test_tod_errors(self, i_error):
l_error = i_error
l_pair = random.choice(self.l_dic)
# Get random chip id
l_chip = l_pair[0]
l_cmd = "PATH=/usr/local/sbin:$PATH putscom -c %s %s %s" % (
l_chip, BMC_CONST.TOD_ERROR_REG, l_error)
console = self.cv_SYSTEM.sys_get_ipmi_console()
console.run_command("dmesg -C")
# As of now putscom command to TOD register will fail with return code -1.
# putscom indirectly call getscom to read the value again.
# But getscom to TOD error reg there is no access
# TOD Error reg has only WO access and there is no read access
try:
l_res = console.run_command(l_cmd, timeout=20)
except CommandFailed as cf:
l_res = cf.output
if cf.exitcode == 1:
pass
else:
if any("Kernel panic - not syncing" in line for line in l_res):
print "TOD ERROR Injection-kernel got panic"
elif any("login:"******"System booted to host OS without any kernel panic message"
elif any("Petitboot" in line for line in l_res):
print "System reached petitboot without any kernel panic message"
elif any("ISTEP" in line for line in l_res):
print "System started booting without any kernel panic message"
else:
raise Exception(
"TOD: PSS Hamming distance error injection failed %s",
str(cf))
time.sleep(0.2)
l_res = console.run_command("dmesg")
self.verify_timer_facility_recovery(l_res)
return
##
# @brief This function enables a single core
def host_enable_single_core(self):
self.cv_HOST.host_enable_single_core()
class OpTestFlashBase(unittest.TestCase):
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_SYSTEM = conf.system()
self.cv_BMC = conf.bmc()
self.cv_REST = self.cv_BMC.get_rest_api()
self.cv_HOST = conf.host()
self.cv_IPMI = conf.ipmi()
self.platform = conf.platform()
self.util = OpTestUtil()
self.bmc_type = conf.args.bmc_type
self.bmc_ip = conf.args.bmc_ip
self.bmc_username = conf.args.bmc_username
self.bmc_password = conf.args.bmc_password
self.pupdate_binary = conf.args.pupdate
self.pflash = conf.args.pflash
def validate_side_activated(self):
l_bmc_side, l_pnor_side = self.cv_IPMI.ipmi_get_side_activated()
self.assertIn(BMC_CONST.PRIMARY_SIDE, l_bmc_side, "BMC: Primary side is not active")
# TODO force setting of primary side to BIOS Golden side sensor
self.assertIn(BMC_CONST.PRIMARY_SIDE, l_pnor_side, "PNOR: Primary side is not active")
def get_pnor_level(self):
rc = self.cv_IPMI.ipmi_get_PNOR_level()
print rc
def bmc_down_check(self):
self.assertTrue(self.util.ping_fail_check(self.cv_BMC.host_name), "FSP/BMC keeps on pinging up")
def scp_file(self, src_file_path, dst_file_path):
self.util.copyFilesToDest(src_file_path, self.bmc_username, self.bmc_ip,
dst_file_path, self.bmc_password, "2", BMC_CONST.SCP_TO_REMOTE)
def get_version_tar(self, file_path):
tar = tarfile.open(file_path)
for member in tar.getmembers():
fd = tar.extractfile(member)
content = fd.read()
if "version=" in content:
content = content.split("\n")
content = [x for x in content if "version=" in x]
version = content[0].split("=")[-1]
break
tar.close()
print version
return version
def get_image_version(self, path):
output = self.cv_BMC.run_command("cat %s | grep \"version=\"" % path)
return output[0].split("=")[-1]
def delete_images_dir(self):
try:
self.cv_BMC.run_command("rm -rf /tmp/images/*")
except CommandFailed:
pass
def get_image_path(self, image_version):
retry = 0
while (retry < 20):
image_list = []
try:
image_list = self.cv_BMC.run_command("ls -1 -d /tmp/images/*/ --color=never")
except CommandFailed as cf:
pass
for i in range(0, len(image_list)):
version = self.get_image_version(image_list[i] + "MANIFEST")
if (version == image_version):
return image_list[i]
time.sleep(5)
retry += 1
def get_image_id(self, version):
img_path = self.get_image_path(version)
img_id = img_path.split("/")[-2]
print "Image id for Host image is : %s" % img_id
return img_id
def wait_for_bmc_runtime(self):
self.util.PingFunc(self.bmc_ip, BMC_CONST.PING_RETRY_FOR_STABILITY)
if "SMC" in self.bmc_type:
self.cv_IPMI.ipmi_wait_for_bmc_runtime()
elif "OpenBMC" in self.bmc_type:
self.cv_REST.wait_for_bmc_runtime()
return
class OpTestMtdPnorDriver():
## Initialize this object and also getting the lpar login credentials to use by scp utility
# @param i_bmcIP The IP address of the BMC
# @param i_bmcUser The userid to log into the BMC with
# @param i_bmcPasswd The password of the userid to log into the BMC with
# @param i_bmcUserIpmi The userid to issue the BMC IPMI commands with
# @param i_bmcPasswdIpmi The password of BMC IPMI userid
# @param i_ffdcDir Optional param to indicate where to write FFDC
#
# "Only required for inband tests" else Default = None
# @param i_lparIP The IP address of the LPAR
# @param i_lparuser The userid to log into the LPAR
# @param i_lparPasswd The password of the userid to log into the LPAR with
#
def __init__(self, i_bmcIP, i_bmcUser, i_bmcPasswd,
i_bmcUserIpmi, i_bmcPasswdIpmi, i_ffdcDir=None, i_lparip=None,
i_lparuser=None, i_lparPasswd=None):
self.cv_BMC = OpTestBMC(i_bmcIP, i_bmcUser, i_bmcPasswd, i_ffdcDir)
self.cv_IPMI = OpTestIPMI(i_bmcIP, i_bmcUserIpmi, i_bmcPasswdIpmi,
i_ffdcDir)
self.cv_LPAR = OpTestLpar(i_lparip, i_lparuser, i_lparPasswd, i_bmcIP)
self.util = OpTestUtil()
self.lpar_user = i_lparuser
self.lpar_ip = i_lparip
self.lpar_Passwd = i_lparPasswd
##
# @brief This function has following test steps
# 1. Get lpar information(OS and Kernel information)
# 2. Load the mtd module based on config value
# 3. Check /dev/mtd0 character device file existence on lpar
# 4. Copying the contents of the flash in a file /tmp/pnor
# 5. Getting the /tmp/pnor file into local x86 machine using scp utility
# 6. Remove existing /tmp/ffs directory and
# Clone latest ffs git repository in local x86 working machine
# 7. Compile ffs repository to get fcp utility
# 8. Check existence of fcp utility in ffs repository, after compiling
# 9. Get the PNOR flash contents on an x86 machine using fcp utility
#
# @return BMC_CONST.FW_SUCCESS-success or raise OpTestError-fail
#
def testMtdPnorDriver(self):
# Get OS level
l_oslevel = self.cv_LPAR.lpar_get_OS_Level()
# Get Kernel Version
l_kernel = self.cv_LPAR.lpar_get_kernel_version()
# loading mtd module based on config option
l_config = "CONFIG_MTD_POWERNV_FLASH"
l_module = "mtd"
self.cv_LPAR.lpar_load_module_based_on_config(l_kernel, l_config, l_module)
# Check /dev/mtd0 file existence on lpar
l_cmd = "ls -l /dev/mtd0; echo $?"
l_res = self.cv_LPAR.lpar_run_command(l_cmd)
l_res = l_res.splitlines()
if int(l_res[-1]) == 0:
print "/dev/mtd0 character device file exists on lpar"
else:
l_msg = "/dev/mtd0 character device file doesn't exist on lpar"
print l_msg
raise OpTestError(l_msg)
# Copying the contents of the PNOR flash in a file /tmp/pnor
l_file = "/tmp/pnor"
l_cmd = "cat /dev/mtd0 > %s; echo $?" % l_file
l_res = self.cv_LPAR.lpar_run_command(l_cmd)
l_res = l_res.splitlines()
if int(l_res[-1]) == 0:
print "Fetched PNOR data from /dev/mtd0 into temp file /tmp/pnor"
else:
l_msg = "Fetching PNOR data is failed from /dev/mtd0 into temp file /tmp/pnor"
print l_msg
raise OpTestError(l_msg)
# Getting the /tmp/pnor file into local x86 machine
l_path = "/tmp/"
self.util.copyFilesToDest(l_path, self.lpar_user, self.lpar_ip, l_file, self.lpar_Passwd, "2", BMC_CONST.SCP_TO_LOCAL)
l_list = commands.getstatusoutput("ls -l %s; echo $?" % l_path)
print l_list
l_workdir = "/tmp/ffs"
# Remove existing /tmp/ffs directory
l_res = commands.getstatusoutput("rm -rf %s" % l_workdir)
print l_res
# Clone latest ffs git repository in local x86 working machine
l_cmd = "git clone https://github.com/open-power/ffs/ %s" % l_workdir
l_res = commands.getstatusoutput(l_cmd)
print l_res
if int(l_res[0]) == 0:
print "Cloning of ffs repository is successfull"
else:
l_msg = "Cloning ffs repository is failed"
print l_msg
raise OpTestError(l_msg)
# Compile ffs repository to get fcp utility
l_cmd = "cd %s/; make" % l_workdir
l_res = commands.getstatusoutput(l_cmd)
print l_res
if int(l_res[0]) == 0:
print "Compiling fcp utility is successfull"
else:
l_msg = "Compiling fcp utility is failed"
print l_msg
raise OpTestError(l_msg)
# Check existence of fcp utility in ffs repository, after compiling.
l_cmd = "test -f %s/fcp/x86/fcp" % l_workdir
l_res = commands.getstatusoutput(l_cmd)
print l_res
if int(l_res[0]) == 0:
print "Compiling fcp utility is successfull"
else:
l_msg = "Compiling fcp utility is failed"
print l_msg
raise OpTestError(l_msg)
# Check the PNOR flash contents on an x86 machine using fcp utility
l_cmd = "%s/fcp/x86/fcp -o 0x0 -L %s" % (l_workdir, l_file)
l_res = commands.getstatusoutput(l_cmd)
print l_res[1]
if int(l_res[0]) == 0:
print "Getting PNOR data successfull using fcp utility"
return BMC_CONST.FW_SUCCESS
else:
l_msg = "Getting the PNOR data using fcp utility failed"
print l_msg
raise OpTestError(l_msg)