def test_all_fans_read(self):
'''
Test if all fan dump is returning sane data
'''
Logger.log_testname(name=__name__)
self.assertNotEqual(self.read_fans_cmd, None, "Get Fan cmd not set")
data = self.get_fan_speed()
Logger.info("Fans dump:\n" + data)
self.assertTrue(self.verify_get_fan_speed_output(data),
"Get fan speeds failed")
def compare_pids(self, process_name: str, original_pid: int, new_pid: int) -> int:
if new_pid != original_pid:
Logger.info(
"PID changed in {}: original pid={}; new pid = {}".format(
process_name, original_pid, new_pid
)
)
return 1
else:
return 0
def test_eth0_4088_v6_interface_link_local(self):
"""
Tests eth0 v6 interface
"""
self.set_ifname("eth0.4088")
Logger.log_testname(self._testMethodName)
self.assertEqual(
self.ping_v6_link_local(),
0,
"Ping test for eth0.4088 v6 over link local failed",
)
def test_eth0_v6_slaac_interface(self):
"""
Tests eth0 SLAAC IP
"""
self.set_ifname("eth0")
Logger.log_testname(name=self._testMethodName)
self.assertIn(
self.generate_modified_eui_64_mac_address(),
self.get_ipv6_address().replace(":", ""),
"SLAAC IP doesn't found",
)
def test_fw_entiy_component_cmd_in_ordered_json(self):
Logger.info("FW Upgrade Ordered json= {}".format(self.json))
for item, _x in self._COMPONENTS.items():
# Test for fw entity presence in json
with self.subTest(upgradable_component=item):
self.assertIn(
item,
self.json,
"Upgradable component {} missing in ordered json {} ".
format(item, self.json),
)
def tearDown(self):
#
# Step 6: restore watchdog-kicking daemon process.
#
Logger.info("Restoring watchdog-kicking process")
self.set_start_watchdog_daemon_cmd()
self.assertNotEqual(self.start_watchdog_daemon_cmd, None,
"Starting watchdog daemon controller cmd not set")
for cmd in self.start_watchdog_daemon_cmd:
run_shell_cmd(cmd)
sleep(10) # wait before daemon started back
def get_cpu_utilization(self):
"""
Getting the data containing CPU consumption
"""
self.set_cpu_utilization_cmd()
self.assertNotEqual(
self.cpu_utilization_cmd, None, "Command to get cpu utilization is not set"
)
Logger.info("Executing cmd={}".format(self.cpu_utilization_cmd))
info = os.popen(self.cpu_utilization_cmd)
return info
def test_host_mac(self):
self.set_host_mac()
self.assertNotEqual(self.host_mac_cmd, None,
"Host MAC command not set")
Logger.info("Executing cmd={}".format(self.host_mac_cmd))
info = run_shell_cmd(cmd=self.host_mac_cmd)
self.assertTrue(
mac_verify(info),
"Host MAC is incorrectly formatted to \
{}".format(info),
)
def gpio_shadow_path(self, gpioname, attributes):
self.assertNotEqual(gpioname, None, "GPIO name not set")
gpio_path = os.path.join(BaseGpioTest.GPIO_SHADOW_ROOT, str(gpioname))
self.assertTrue(os.path.exists(gpio_path), "GPIO missing={}".format(gpioname))
Logger.info("GPIO name={} present".format(gpioname))
for item, _value in attributes.items():
path = os.path.join(gpio_path, str(item))
self.assertTrue(
os.path.exists(path),
"GPIO attribute missing={} attribute={}".format(gpioname, str(item)),
)
def test_fw_entiy_priority_in_ordered_json(self):
Logger.info("FW Upgrade Ordered json= {}".format(self.json))
for item, attributes in self._COMPONENTS.items():
# Test for fw entity priority set in json
with self.subTest(upgradable_component=item):
self.assertEqual(
attributes[0],
self.json.get(item).get("priority"),
"Component {} contains priority expected={} found={} ".
format(item, attributes[0],
self.json.get(item).get("priority")),
)
def run_ping(self, cmd):
self.assertNotEqual(cmd, None, "run_ping cmd not set")
if cmd != "":
Logger.debug("Executing: " + str(cmd))
f = subprocess.Popen(cmd, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
err = f.communicate()[1]
if len(err) > 0:
raise Exception(err)
return f.returncode
return -1
def read_data_from_filepath(path):
'''
Helper method to read file from a file
'''
try:
handle = open(path, "r")
data = handle.readline().rstrip()
handle.close()
Logger.debug("Reading path={} data={}".format(path, data))
return data
except Exception:
raise ("Failed to read path={}".format(path))
def test_all_fans_read(self):
"""
Test if all fan dump is returning sane data
"""
# Sleep needed to test and make sure that we won't run
# into accessing sysfs path issue causing test to fail
time.sleep(2)
Logger.log_testname(self._testMethodName)
self.assertNotEqual(self.read_fans_cmd, None, "Get Fan cmd not set")
data = self.get_fan_speed()
Logger.info("Fans dump:\n" + data)
self.assertTrue(self.verify_get_fan_speed_output(data), "Get fan speeds failed")
def get_from_endpoint(self, endpointname=None):
self.assertNotEqual(endpointname, None, "Endpoint name not set")
cmd = BaseRestEndpointTest.CURL_CMD6.format(endpointname)
Logger.info("Executing cmd={}".format(cmd))
try:
# Send request to REST service to see if it's alive
handle = urlopen(cmd)
data = handle.read()
Logger.info("REST request returned data={}".format(data.decode('utf-8')))
return data.decode('utf-8')
except Exception as ex:
raise("Failed to perform REST request for cmd={}".format(cmd))
def test_fw_entiy_upgrade_cmd_in_ordered_json(self):
Logger.info("FW Upgrade Ordered json= {}".format(self.json))
for item, attributes in self._COMPONENTS.items():
# Test for correct command in json
with self.subTest(upgradable_component=item):
self.assertEqual(
attributes[1],
self.json.get(item).get("upgrade_cmd"),
"Component {} contains upgrade_cmd expected={} found={} ".
format(item, attributes[1],
self.json.get(item).get("upgrade_cmd")),
)
def upgrade_components(self, components_to_upgrade, logging=False):
"""
main method to check and upgrade the components
"""
for component in components_to_upgrade:
# Start timestamp
start = time.perf_counter()
entity = component["entity"]
ret = -1
if component["upgrade_needed"]:
try:
if not self.bmc_ssh_session.session.isalive():
self.fail("remote ssh session broke!")
filename = self.remote_bin_path + "/" + self.json[entity][UFW_NAME]
cmd_to_execute = self.json[entity][UFW_CMD]
cmd_to_execute = cmd_to_execute.format(filename=filename)
if logging:
self.print_line(
"Updating: {} ".format(entity),
"left",
".",
endline=" ",
has_append=True,
)
self.send_command_to_UUT(self.STOP_FSCD)
self.send_command_to_UUT(self.STOP_WDT)
self.send_command_to_UUT(cmd_to_execute, prompt=False)
ret = self.bmc_ssh_session.session.expect_exact(
self.expected_keyword, timeout=self.upgrading_timeout[entity]
)
except pexpect.exceptions.TIMEOUT:
ret = -1
if logging:
print("Timed out")
except Exception as e:
print(e)
# End timestamp
end = time.perf_counter()
component["execution_time"] = end - start
if ret >= 0 and ret <= self.num_last_failed_expected_key:
if logging:
print("Failed")
component["upgrade_status"] = False
component["result"] = self.receive_command_output_from_UUT()
Logger.error("{}: Upgrading failed!".format(component))
else:
if logging:
print("Done")
# flush the rest log.
self.flush_session_contents()
time.sleep(10) # delay for the current process to be done
def kill_watchdog_daemon(self):
"""
Helper method to kill watchdog daemon: FSCD, fand, healthd
"""
self.set_kill_watchdog_daemon_cmd()
self.assertNotEqual(
self.kill_watchdog_daemon_cmd,
None,
"Kill watchdog daemon controller cmd not set",
)
Logger.info("killing watchdog-kicking daemon processes")
for cmd in self.kill_watchdog_daemon_cmd:
run_shell_cmd(cmd)
def test_eth0_v6_slaac_interface(self):
"""
Tests eth0 SLAAC IP
"""
self.set_ifname("eth0")
Logger.log_testname(name=self._testMethodName)
addr = ipaddress.ip_address(self.get_ipv6_address())
haystack = addr.exploded.replace(":", "")
self.assertIn(
self.generate_modified_eui_64_mac_address(),
haystack,
"SLAAC IP doesn't found",
)
def get_ipv6_address(self):
"""
Get inet6 address with highest length of a given interface
overriding this method of BaseInterfaceTest class because we want
to have inet6 address with highest length
"""
out = self.get_ip_addr_output_inet6()
# trying to find inet6 address with highest length
ipv6 = ""
for value in out[1:]:
if len(value.split("/")[0]) > len(ipv6):
ipv6 = value.split("/")[0]
Logger.debug("Got ip address for " + str(self.ifname))
return ipv6.lower()
def get_ipv4_address(self):
"""
Get IPv4 address of a given interface
"""
f = subprocess.Popen(['ip', 'addr', 'show', self.ifname.encode('utf-8')],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = f.communicate()
if (len(out) == 0 or len(err) != 0):
raise Exception("Device " + self.ifname.encode('utf-8') + " does not exist [FAILED]")
out = out.decode('utf-8')
ipv4 = out.split("inet ")[1].split("/")[0]
Logger.debug("Got ip address for " + str(self.ifname))
return ipv4
def test_watchdog_kick(self):
self.kill_watchdog_daemon()
#
# Step 5: check if watchdog can be kicked properly. The bmc will
# reboot if watchdog cannot be kicked properly.
#
interval = 6
iterations = 6
Logger.info("testing if watchdog can be kicked properly")
for i in range(iterations):
self.wdtUtils.kick_watchdog()
Logger.info('[%d/%d] watchdog kicked. Sleeping for %d seconds' %
(i, iterations, interval))
sleep(interval)
self.assertEqual(i, iterations-1,
"Failed to kick watchdog for {} iterations".format(iterations))
def setUp(self):
Logger.start(name=self._testMethodName)
self.fans = [0, 1, 2, 3, 4, 5, 6, 7]
self.pwms = {0: [2, 3, 4, 5], 1: [0, 1, 6, 7]}
self.names = {
"Fan 1 Front": 0,
"Fan 1 Rear": 1,
"Fan 2 Front": 2,
"Fan 2 Rear": 3,
"Fan 3 Front": 4,
"Fan 3 Rear": 5,
"Fan 4 Front": 6,
"Fan 4 Rear": 7,
}
self.kill_fan_ctrl_cmd = ["/usr/bin/sv stop fscd"]
self.start_fan_ctrl_cmd = ["/usr/bin/sv start fscd"]
def reconnect_to_remote_host(self, timeout=30, logging=False):
"""
reconnect to remote DUT until the timeout is expired
"""
if logging:
self.print_line(
"Reconnecting to DUT ", "left", ".", endline=" ", has_append=True
)
Logger.info("Reconnecting to DUT ...")
if not self.wait_until(lambda: self.connect_to_remote_host(), timeout):
print("Failed")
return False
time.sleep(self.scm_boot_time) # waiting for SCM to get ready
if logging:
print("Done")
return True
def power_host_on(self):
retry = 5
for i in range(retry):
# If host was already on, script takes care of just returning on
cmd = "/usr/local/bin/wedge_power.sh on"
data = run_shell_cmd(cmd)
Logger.info("[FSCD Testing] Try {} Executing cmd= [{}]".format(i, cmd))
Logger.info("[FSCD Testing] Received data= [{}]".format(data))
time.sleep(5)
if self.is_host_on():
return
self.assertTrue(
self.is_host_on(),
"[FSCD Testing] Retry for {} times"
" and host failed to power on".format(retry),
)
def setUp(self, config=None, test_data_path=None):
"""
Series of tests that are driven by changing the temperature sensors
"""
Logger.start(name=self._testMethodName)
self.assertNotEqual(config, None, "FSC TEST config needs to be set")
self.assertNotEqual(test_data_path, None, "FSC TEST data path needs to be set")
# Backup original config
run_shell_cmd("cp /etc/fsc-config.json /etc/fsc-config.json.orig")
# Copy test config to fsc-config and restart fsc
run_shell_cmd(
"cp {}/{} /etc/fsc-config.json".format(test_data_path, str(config))
)
self.restart_fscd()
def get_from_endpoint(self, endpointname=None):
self.assertNotEqual(endpointname, None, "Endpoint name not set")
endpoint = BaseRestEndpointTest.CURL_CMD6.format(endpointname)
Logger.info("Requesting GET {endpoint}".format(endpoint=endpoint))
try:
# Send request to REST service to see if it's alive
handle = urlopen(endpoint)
data = handle.read()
Logger.info("REST request returned data={}".format(data.decode("utf-8")))
return data.decode("utf-8")
except Exception as ex:
raise BaseRestEndpointException(
"Failed to request GET {endpoint}, Exception={ex}".format(
endpoint=endpoint,
ex=repr(ex),
)
) from None
def setUp(self):
Logger.start(name=self._testMethodName)
if running_systemd():
Logger.info("Running the systemd variant")
self.kill_fan_ctrl_cmd = [
"/bin/systemctl stop fscd",
"/usr/local/bin/wdtcli stop",
]
self.start_fan_ctrl_cmd = ["/bin/systemctl start fscd"]
else:
self.kill_fan_ctrl_cmd = [
"/usr/bin/sv stop fscd",
"/usr/local/bin/wdtcli stop",
]
self.start_fan_ctrl_cmd = ["/usr/bin/sv start fscd"]
self.read_fans_cmd = "/usr/local/bin/get_fan_speed.sh"
self.write_fans_cmd = "/usr/local/bin/set_fan_speed.sh"
def verify_binary_checksum_on_remote_target(self, logging=False):
"""
verify that the firmware binary hash on UUT matches
the ones in the json
"""
if logging:
self.print_line(
"Checking binary on UUT ",
"left",
".",
self.MAX_LINE_LEN - 10,
endline=" ",
)
Logger.info("Checking binary hash on UUT ... ")
sha1sum_cmd = "sha1sum {} | cut -d ' ' -f 1"
md5sum_cmd = "md5sum {} | cut -d ' ' -f 1"
for fw_entity in self.json:
filename = os.path.join(
self.remote_bin_path, self.json[fw_entity][UFW_NAME]
)
matchingHash = False
if self.json[fw_entity][UFW_HASH] == HashType.SHA1SUM.value:
test_cmd = sha1sum_cmd.format(filename)
elif self.json[fw_entity][UFW_HASH] == HashType.MD5SUM.value:
test_cmd = md5sum_cmd.format(filename)
else:
self.fail("unknow hash type on component {}".format(fw_entity))
self.send_command_to_UUT(test_cmd)
binary_hash = self.receive_command_output_from_UUT(only_last=True)
# verify hash string from UUT
matchingHash = binary_hash == self.json[fw_entity][UFW_HASH_VALUE]
self.assertTrue(
matchingHash,
"firmware component {} missmatch for file {}".format(
fw_entity, filename
),
)
if logging:
print("Done")
def verify_get_fan_speed_output(self, info):
"""
Parses get_fan_speed.sh output
and test it is in the expected format
Arguments: info = dump of get_fan_speed.sh
"""
Logger.info("Parsing Fans dump: {}".format(info))
info = info.split("\n")
goodRead = ["Fan", "RPMs:", "%"]
for line in info:
if len(line) == 0:
continue
for word in goodRead:
if word not in line:
return False
val = line.split(":")[1].split(",")
if len(val) != 3:
return False
return True
def connect_to_remote_host(self, logging=False):
"""
method to open ssh session to perform external testing
"""
if logging:
self.print_line(
"Connecting to UUT ", "left", ".", endline=" ", has_append=True
)
Logger.info("Connecting to UUT ...")
try:
self.bmc_ssh_session = OpenBMCSSHSession(self.bmc_hostname)
self.bmc_ssh_session.connect()
self.bmc_ssh_session.login() # connect to BMC UUT
self.bmc_ssh_session.session.prompt(timeout=20) # wait for prompt
if logging:
print("Done")
return True
except Exception:
return False
