def test_bmc_is_accessible(bmc_ip, bmc_user, bmc_passwd):
"""Check if BMC is accessible through KCS or LAN."""
channel_interface = Conf.get(
consts.SSPL_CONFIG_INDEX, "BMC_INTERFACE>default", "system")
if channel_interface == "system":
# Check BMC is accessible through KCS
cmd = "sudo ipmitool channel info"
expected_channel = "KCS"
channel_found = None
channel_protocol = "Channel Protocol Type"
res_op, res_err, res_rc = SimpleProcess(cmd).run()
if res_rc == 0:
res_op = res_op.decode()
search_res = re.search(
r"%s[\s]+:[\s]+(\w+)(.*)" % channel_protocol, res_op)
if search_res:
channel_found = search_res.groups()[0]
if expected_channel != channel_found:
logger.critical(
"UNEXPECTED BMC CHANNEL TYPE FOUND. Expected: '%s' Found: '%s'" %(
expected_channel, channel_found))
else:
res_err = res_err.decode()
kcs_errors = ("could not find inband device", "driver timeout")
if not any(err for err in kcs_errors if err in res_err):
raise Exception("BMC is NOT accessible through KCS - ERROR: %s" % res_err)
elif channel_interface == "lan":
# Check BMC is accessible through LAN
subcommand = "channel info"
cmd = "sudo ipmitool -H %s -U %s -P %s -I lan %s" %(
bmc_ip, bmc_user, bmc_passwd, subcommand)
_, res_err, res_rc = SimpleProcess(cmd).run()
if res_rc != 0:
raise Exception("BMC is NOT accessible over lan - ERROR: %s" % res_err.decode())
def test_iem_receive_with_valid_cmdline_arguments(self):
# send data to receive
SimpleProcess(self.send_with_valid_cmdline_args).run()
cmd = self.valid_recv
output, _, rc = SimpleProcess(cmd).run()
self.assertTrue('critical' in output.decode('utf-8'))
self.assertEqual(rc, 0)
def get_server_details():
"""Returns a dictionary of server information using ipmitool.
Grep 'FRU device description on ID 0' information from the output
of 'ipmitool fru print'. Server details includes Hostname, Board and
Product information.
"""
fru_info = {
"Host": socket.getfqdn(),
"Board Mfg": None,
"Board Product": None,
"Board Part Number": None,
"Product Name": None,
"Product Part Number": None
}
cmd = "ipmitool fru print"
prefix = "FRU Device Description : Builtin FRU Device (ID 0)"
search_res = ""
res_op, _, res_rc = SimpleProcess(cmd).run()
if isinstance(res_op, bytes):
res_op = res_op.decode("utf-8")
if res_rc == 0:
# Get only 'FRU Device Description : Builtin FRU Device (ID 0)' information
search_res = re.search(
r"((.*%s[\S\n\s]+ID 1\)).*)|(.*[\S\n\s]+)" % prefix, res_op)
if search_res:
search_res = search_res.group()
for key in fru_info.keys():
if key in search_res:
device_desc = re.search(r"%s[\s]+:[\s]+([\w-]+)(.*)" % key, res_op)
if device_desc:
value = device_desc.groups()[0]
fru_info.update({key: value})
return fru_info
def set_lshw_input_data(self):
"""
KvStoreFactory can not accept a dictionary as direct input and output
It will support only JSON, YAML, TOML, INI, PROPERTIES files. So here
we are fetching the lshw data and adding that to a file for further
execution.
"""
input_file = None
output_file = None
response, err, returncode = SimpleProcess("lshw -json").run()
if returncode:
msg = f"Failed to capture Node support data. Error:{str(err)}"
logger.error(self.log.svc_log(msg))
raise ResourceMapError(errno.EINVAL, msg)
try:
with open(LSHW_FILE, 'w+') as fp:
json.dump(json.loads(response.decode("utf-8")), fp, indent=4)
with open(MANIFEST_OUTPUT_FILE, 'w+') as fp:
json.dump({}, fp, indent=4)
input_file = KvStoreFactory.get_instance(
f'json://{LSHW_FILE}').load()
output_file = KvStoreFactory.get_instance(
f'json://{MANIFEST_OUTPUT_FILE}')
except Exception as e:
msg = "Error in getting {0} file: {1}".format(LSHW_FILE, e)
logger.error(self.log.svc_log(msg))
raise ResourceMapError(errno.EINVAL, msg)
return input_file, output_file
def get_devices(self):
output, err, returncode = SimpleProcess(
f"mdadm --detail --test {self.raid}").run()
if returncode == 0:
output = output.decode()
else:
output = err.decode()
devices = []
for state in re.findall(r"^\s*\d+\s*\d+\s*\d+\s*\d+\s*(.*)", output,
re.MULTILINE):
device = {}
device["state"] = state
device["identity"] = {}
path = re.findall(r"\/dev\/(.*)", state)
if path:
device["identity"]["path"] = f"/dev/{path[0]}"
output, _, returncode = SimpleProcess(
f'smartctl -i {device["identity"]["path"]} --json').run()
if returncode == 0:
output = json.loads(output)
try:
device["identity"]["serialNumber"] = output[
"serial_number"]
except KeyError:
device["identity"]["serialNumber"] = "None"
else:
device["identity"]["serialNumber"] = "None"
devices.append(device)
return devices
def get_rabbitmq_cluster_nodes(self):
cluster_nodes = None
if self.rabbitmq_major_release == '3' and \
self.rabbitmq_maintenance_release == '8':
rmq_cluster_status_cmd = '/usr/sbin/rabbitmqctl cluster_status' + \
' --formatter json'
output, error, returncode = SimpleProcess(rmq_cluster_status_cmd).run()
try:
rabbitmq_cluster_status = json.loads(output)
except Exception:
raise SetupError(
errno.EINVAL,
"RabbitMQ cluster status is not okay! \n, status : %s",
output)
if returncode:
raise SetupError(returncode, error)
running_nodes = rabbitmq_cluster_status['running_nodes']
for i, node in enumerate(running_nodes):
running_nodes[i] = node.replace('rabbit@', '')
cluster_nodes = " ".join(running_nodes)
elif self.rabbitmq_version == '3.3.5':
rmq_cluster_status_cmd = "rabbitmqctl cluster_status"
output, error, returncode = SimpleProcess(rmq_cluster_status_cmd).run()
cluster_nodes = re.search(
r"running_nodes,\['rabbit@(?P<cluster_nodes>.+?)'\]",
output.decode()).groupdict()["cluster_nodes"]
else:
raise SetupError(
errno.EINVAL,
"This RabbitMQ version : %s is not supported",
self.rabbitmq_version)
return cluster_nodes
def get_service_info_from_rpm(service, prop):
"""
Get specified service property from its corrosponding RPM.
eg. (kafka.service,'LICENSE') -> 'Apache License, Version 2.0'
"""
# TODO Include service execution path in systemd_path_list
systemd_path_list = [
"/usr/lib/systemd/system/", "/etc/systemd/system/"
]
result = "NA"
for path in systemd_path_list:
# unit_file_path represents the path where
# systemd service file resides
# eg. kafka service -> /etc/systemd/system/kafka.service
unit_file_path = path + service
if os.path.isfile(unit_file_path):
# this command will return the full name of RPM
# which installs the service at given unit_file_path
# eg. /etc/systemd/system/kafka.service -> kafka-2.13_2.7.0-el7.x86_64
command = " ".join(["rpm", "-qf", unit_file_path])
command = shlex.split(command)
service_rpm, _, ret_code = SimpleProcess(command).run()
if ret_code != 0:
return result
try:
service_rpm = service_rpm.decode("utf-8")
except AttributeError:
return result
# this command will extract specified property from given RPM
# eg. (kafka-2.13_2.7.0-el7.x86_64, 'LICENSE') -> 'Apache License, Version 2.0'
command = " ".join([
"rpm", "-q", "--queryformat", "%{" + prop + "}",
service_rpm
])
command = shlex.split(command)
result, _, ret_code = SimpleProcess(command).run()
if ret_code != 0:
return result
try:
# returned result should be in byte which need to be decoded
# eg. b'Apache License, Version 2.0' -> 'Apache License, Version 2.0'
result = result.decode("utf-8")
except AttributeError:
return result
break
return result
def test_chassis_selftest(args):
"""Check chassis selttestsel is passed."""
cmd = "ipmitool chassis selftest"
expected_res = "Self Test Results : passed"
res_op, res_err, res_rc = SimpleProcess(cmd).run()
if res_rc == 0:
res_op = res_op.decode()
if expected_res not in res_op:
assert False, res_op
else:
raise Exception("ERROR: %s" % res_err.decode())
def test_iem_receive_output_to_log_file(self):
# send data to receive
SimpleProcess(self.send_with_valid_cmdline_args).run()
cmd = self.valid_recv_log_file
_, _, rc = SimpleProcess(cmd).run()
self.assertEqual(rc, 0)
file_output, _, _ = SimpleProcess("ls /tmp/iem_receive.log").run()
# Check if file is present
self.assertTrue('iem_receive.log' in file_output.decode('utf-8'))
# Check if an expected word is present in file
with open('/tmp/iem_receive.log') as fd:
data = ' '.join(fd.readlines())
self.assertTrue('cmd_line_message' in data)
def _send_command(self, command, fail_on_error=True):
"""
Execute command and retrun response
Parameters:
command: shell command to execute
fail_on_error: Set to False will ignore command failure
"""
print(f"Executing: {command}")
output, error, returncode = SimpleProcess(command).run()
if fail_on_error and returncode != 0:
raise SetupError(returncode,
"ERROR: Command '%s' failed with error\n %s",
command, error)
return output.decode('utf-8')
def get_service_uptime(service_name):
"""Get service uptime."""
uptime = "NA"
cmd = f"systemctl status {service_name} "
output, error, returncode = SimpleProcess(cmd).run()
if returncode != 0:
logger.error("get_service_timestamp: CMD %s failed to get timestamp "
"due to error: %s" % (cmd, error))
else:
output = output.decode('utf-8')
timestamp_status = r"Active:(.*) since (.*)"
for line in output.splitlines():
status_line = re.search(timestamp_status, line)
if status_line:
uptime = status_line.group(2).strip()
return uptime
def get_manufacturer_name():
"""Returns node server manufacturer name.
Example: Supermicro, Intel Corporation, DELL Inc
"""
manufacturer = ""
cmd = "ipmitool bmc info"
res_op, _, res_rc = SimpleProcess(cmd).run()
if isinstance(res_op, bytes):
res_op = res_op.decode("utf-8")
if res_rc == 0:
search_res = re.search(r"Manufacturer Name[\s]+:[\s]+([\w]+)(.*)",
res_op)
if search_res:
manufacturer = search_res.groups()[0]
return manufacturer
def test_ipmi_version(args):
"""Check for expected IPMI version."""
# Check IPMI version
ipmi_ver_cmd = "ipmitool mc info" # IPMI Version : 2.0
version_found = None
res_op, res_err, res_rc = SimpleProcess(ipmi_ver_cmd).run()
if res_rc == 0:
res_op = res_op.decode()
search_res = re.search(r"%s[\s]+([\w.]+)(.*)" % IPMI_VERSION, res_op)
if search_res:
version_found = search_res.groups()[0]
else:
raise Exception("ERROR: %s" % res_err.decode())
if not (float(search_res.groups()[0]) >= MIN_REQUIRED_IPMI_VERSION):
print("VERSION MISMATCH WITH IPMIT")
print("Expected: %s" % REQUIRED_IPMI_VERSION)
print("Found: %s" % version_found)
def get_health(self):
if self._is_local_drive():
smartctl = "sudo smartctl"
else:
smartctl = "sudo smartctl -d scsi"
# Get smart availability attributes
cmd = f"{smartctl} -i {self.device}"
response, _, _ = SimpleProcess(cmd).run()
response = response.decode()
health_data = {}
if re.findall(
"SMART support is: Available - device has SMART capability",
response):
health_data["SMART_available"] = True
else:
health_data["SMART_available"] = False
if re.findall("SMART support is: Enabled", response):
health_data["SMART_support"] = "Enabled"
else:
health_data["SMART_support"] = "NA"
# Get smart health attributes
cmd = f"{smartctl} --all {self.device} --json"
response, _, _ = SimpleProcess(cmd).run()
response = json.loads(response)
try:
smart_test_status = "PASSED" if response['smart_status'][
'passed'] else "FAILED"
except KeyError:
smart_test_status = "NA"
health_data["SMART_health"] = smart_test_status
smart_required_attributes = set([
"Reallocated_Sector_Ct", "Spin_Retry_Count",
"Current_Pending_Sector", "Offline_Uncorrectable"
])
try:
for attribute in response["ata_smart_attributes"]["table"]:
try:
if attribute["name"] in smart_required_attributes:
health_data[
attribute["name"]] = attribute["raw"]["value"]
except KeyError:
pass
except KeyError:
pass
return health_data
def test_ipmitool_version(args):
"""Check for expected ipmitool & IPMI v2 compliant."""
# Check ipmitool version
tool_ver_cmd = "ipmitool -V" # ipmitool version 1.8.18
version_found = None
res_op, res_err, res_rc = SimpleProcess(tool_ver_cmd).run()
if res_rc == 0:
res_op = res_op.decode()
search_res = re.search(r"%s[\s]+([\w.]+)(.*)" % IPMITOOL_VERSION,
res_op)
if search_res:
version_found = search_res.groups()[0]
else:
raise Exception("ERROR: %s" % res_err.decode())
if not (version_found >= MIN_REQUIRED_IPMITOOL_VERSION):
print("VERSION MISMATCH WITH IPMITOOL")
print("Expected: %s" % MIN_REQUIRED_IPMITOOL_VERSION)
print("Found: %s" % version_found)
def validate_nw_cable_connection(self, interfaces):
"""Check network interface links are up.
0 - Cable disconnected
1 - Cable connected
"""
if not isinstance(interfaces, list):
raise SetupError(
errno.EINVAL, "%s - validation failure. %s", self.name,
"Expected list of interfaces. Received, %s." % interfaces)
for interface in interfaces:
cmd = "cat /sys/class/net/%s/carrier" % interface
output, error, rc = SimpleProcess(cmd).run()
output = output.decode().strip()
if output == "0":
raise SetupError(
errno.EINVAL, "%s - validation failure. %s", self.name,
"Network interface cable is disconnected - %s." %
interface)
def test_bmc_firmware_version(args):
"""Check if BMC firmware version is 1.71."""
cmd = "ipmitool bmc info" # Firmware Revision : 1.71
expected_ver = "1.71"
version_found = None
res_op, res_err, res_rc = SimpleProcess(cmd).run()
if res_rc == 0:
res_op = res_op.decode()
search_res = re.search(r"%s[\s]+:[\s]+([\w.]+)(.*)" % FIRMWARE_VERSION,
res_op)
if search_res:
version_found = search_res.groups()[0]
else:
raise Exception("ERROR: %s" % res_err.decode())
if expected_ver != version_found:
print("UNEXPECTED BMC FIRMWARE VERSION FOUND.")
print("Expected: %s" % expected_ver)
print("Found: %s" % version_found)
def get_psus():
response, _, _ = SimpleProcess("dmidecode -t 39").run()
matches = re.findall(
"System Power Supply|Power Unit Group:.*|"
"Location:.*|Name:.*|Serial Number:.*|"
"Max Power Capacity:.*|Status: .*|"
"Plugged:.*|Hot Replaceable:.*", response.decode())
psus = []
stack = []
while matches:
item = matches.pop()
while item != "System Power Supply":
stack.append(item)
item = matches.pop()
psu = {}
while stack:
key, value = stack.pop().strip().split(":")
psu[key] = value.strip()
psus.append(psu)
return psus
def test_bmc_is_accessible(args):
"""Check if BMC is accessible through KCS or LAN."""
channel_interface = Conf.get(SSPL_CONF, "BMC_INTERFACE>default", 'system')
if channel_interface == "system":
# Check BMC is accessible through KCS
cmd = "sudo ipmitool channel info"
expected_channel = "KCS"
channel_found = None
res_op, res_err, res_rc = SimpleProcess(cmd).run()
if res_rc == 0:
res_op = res_op.decode()
search_res = re.search(
r"%s[\s]+:[\s]+(\w+)(.*)" % CHANNEL_PROTOCOL, res_op)
if search_res:
channel_found = search_res.groups()[0]
if expected_channel != channel_found:
print("UNEXPECTED BMC CHANNEL TYPE FOUND.")
print("Expected: %s" % expected_channel)
print("Found: %s" % channel_found)
else:
res_err = res_err.decode()
kcs_errors = ("could not find inband device", "driver timeout")
if not any(err for err in kcs_errors if err in res_err):
raise Exception(
"BMC is NOT accessible through KCS - ERROR: %s" % res_err)
elif channel_interface == "lan":
# Check BMC is accessible through LAN
subcommand = "channel info"
bmc_ip = Conf.get(GLOBAL_CONF, BMC_IP_KEY)
bmc_user = Conf.get(GLOBAL_CONF, BMC_USER_KEY)
bmc_secret = Conf.get(GLOBAL_CONF, BMC_SECRET_KEY)
bmc_key = Cipher.generate_key(MACHINE_ID, "server_node")
bmc_passwd = Cipher.decrypt(bmc_key,
bmc_secret.encode("utf-8")).decode("utf-8")
cmd = "sudo ipmitool -H %s -U %s -P %s -I lan %s" % (
bmc_ip, bmc_user, bmc_passwd, subcommand)
res_op, res_err, res_rc = SimpleProcess(cmd).run()
if res_rc != 0:
raise Exception("BMC is NOT accessible over lan - ERROR: %s" %
res_err.decode())
def test_sel_version(args):
"""Check for expected SEL v2 compliant."""
# Check IPMI SEL compliance is >= v2
sel_ver_cmd = "ipmitool sel info" # Version : 1.5 (v1.5, v2 compliant)
res_op, res_err, res_rc = SimpleProcess(sel_ver_cmd).run()
if res_rc == 0:
res_op = res_op.decode()
search_res = re.search(
r"%s[\s]+:[\s]+([\w.]+)[\s]+\(([\w.,\s]+)\)(.*)" % SEL_VERSION,
res_op)
if search_res:
if not (float(search_res.groups()[0]) >= MIN_REQUIRED_SEL_VERSION) and \
"v2" not in search_res.groups()[1]:
# Fail if sel complinace is not >= v2
print("SEL IS NOT V2 COMPLIANT.")
print("Minimum required sel version: %s" %
MIN_REQUIRED_SEL_VERSION)
print("Found: %s" % search_res.groups()[1])
assert (False)
else:
raise Exception("ERROR: %s" % res_err.decode())
def test_sensor_availability(args):
"""Fail if any expected sensor is not detected by ipmitool."""
found_all_sensors = True
sensors = [
"Voltage", "Temperature", "Power Supply", "Drive Slot / Bay", "Fan"
]
# Get manufacturer name
server_info = get_server_details()
manufacturer = server_info["Board Mfg"]
for sensor in sensors:
cmd = ["ipmitool", "sdr", "type", sensor]
res_op, res_err, res_rc = SimpleProcess(cmd).run()
if res_rc == 0:
res_op = res_op.decode().replace("\n", "")
if not res_op:
found_all_sensors = False
print(
"'%s' sensor is not seen in %s node server. Server Information: %s"
% (sensor, manufacturer, server_info))
else:
raise Exception("ERROR: %s" % res_err.decode())
assert (found_all_sensors == True)
def _run_ipmitool_subcommand(self, subcommand, grep_args=None):
"""Executes ipmitool sub-commands, and optionally greps the output."""
self.ACTIVE_IPMI_TOOL = self.IPMITOOL
host_conf_cmd = ""
# Set ipmitool to ipmisimtool if activated.
if os.path.exists(f"{DATA_PATH}/server/activate_ipmisimtool"):
cmd = self.IPMISIMTOOL + " sel info"
_, _, retcode = SimpleProcess(cmd).run()
if retcode in [0, 2]:
self.ACTIVE_IPMI_TOOL = self.IPMISIMTOOL
logger.debug("IPMI simulator is activated.")
# Fetch channel info from config file and cache.
_channel_interface = Conf.get(
SSPL_CONF, "%s>%s" % (BMC_INTERFACE, BMC_CHANNEL_IF))
_active_interface = store.get(BMCInterface.ACTIVE_BMC_IF.value, None)
if isinstance(_active_interface, bytes):
_active_interface = _active_interface.decode()
# Set host_conf_cmd based on channel info.
if (self.ACTIVE_IPMI_TOOL != self.IPMISIMTOOL
and _active_interface in BMCInterface.LAN_IF.value):
bmc_ip = Conf.get(GLOBAL_CONF, BMC_IP_KEY, '')
bmc_user = Conf.get(GLOBAL_CONF, BMC_USER_KEY, 'ADMIN')
bmc_secret = Conf.get(GLOBAL_CONF, BMC_SECRET_KEY, 'ADMIN')
decryption_key = encryptor.gen_key(MACHINE_ID,
ServiceTypes.SERVER_NODE.value)
bmc_pass = encryptor.decrypt(decryption_key, bmc_secret, self.NAME)
host_conf_cmd = BMCInterface.LAN_CMD.value.format(
_active_interface, bmc_ip, bmc_user, bmc_pass)
# generate the final cmd and execute on shell.
command = " ".join([self.ACTIVE_IPMI_TOOL, host_conf_cmd, subcommand])
command = shlex.split(command)
out, error, retcode = SimpleProcess(command).run()
# Decode bytes encoded strings.
if not isinstance(out, str):
out = out.decode(self.IPMI_ENCODING)
if not isinstance(error, str):
error = error.decode(self.IPMI_ENCODING)
# Grep the output as per grep_args provided.
if grep_args is not None and retcode == 0:
final_list = []
for l in out.split('\n'):
if re.search(grep_args, l) is not None:
final_list += [l]
out = '\n'.join(final_list)
# Assign error_msg to err from output
if retcode and not error:
out, error = error, out
# Remove '\n' from error, for matching errors to error stings.
if error:
error = error.replace('\n', '')
return out, error, retcode
