def test_sensor_unrecoverable_failure_alert(args):
"""
Check for fault alert on unrecoverable sensor module failure.
Simulate sensor failure
Verify fault alert on unrecoverable sensor module
Verify sensor module is not running
"""
check_sspl_ll_is_running()
simulate_NodeHWsensor_failure()
# Wait until know its unrecoverable error
ingressMsg = get_fru_response(
resource_type,
resource_id,
ingress_msg_type="sensor_response_type",
timeout=get_maximum_recovery_time(resource_id),
alert_type="fault")
sensor_msg = ingressMsg.get("sensor_response_type")
assert sensor_msg
assert sensor_msg.get("alert_type") == "fault"
assert sensor_msg.get("severity") == "critical"
assert sensor_msg.get("host_id")
info = sensor_msg.get("info")
assert info
assert info.get("site_id")
assert info.get("node_id")
assert info.get("rack_id")
assert info.get("event_time")
assert info.get("impact")
assert info.get("recommendation") == "Restart SSPL service"
assert info.get("description")
assert info.get("resource_type") == resource_type
assert info.get("resource_id") == resource_id
execute_module_thread_operation(action="status",
thread_response="Status: Halted")
def test_node_disk_module_actuator(agrs):
check_sspl_ll_is_running()
instance_id = "*"
resource_type = "node:fru:disk"
send_node_controller_message_request(UUID, "NDHW:%s" % resource_type,
instance_id)
ingressMsg = get_fru_response(resource_type, instance_id)
assert (ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID)
disk_actuator_msg = ingressMsg.get("actuator_response_type")
assert (disk_actuator_msg is not None)
assert (disk_actuator_msg.get("alert_type") is not None)
assert (disk_actuator_msg.get("severity") is not None)
assert (disk_actuator_msg.get("host_id") is not None)
assert (disk_actuator_msg.get("info") is not None)
assert (disk_actuator_msg.get("instance_id") == instance_id)
disk_actuator_info = disk_actuator_msg.get("info")
assert (disk_actuator_info.get("site_id") is not None)
assert (disk_actuator_info.get("node_id") is not None)
assert (disk_actuator_info.get("rack_id") is not None)
assert (disk_actuator_info.get("resource_type") is not None)
assert (disk_actuator_info.get("event_time") is not None)
assert (disk_actuator_info.get("resource_id") is not None)
disk_specific_infos = disk_actuator_msg.get("specific_info")
assert (disk_specific_infos is not None)
if disk_actuator_info.get("resource_id") == "*":
for disk_specific_info in disk_specific_infos:
assert (disk_specific_info is not None)
if disk_specific_info.get("ERROR"):
# Skip any validation on specific info if ERROR seen on FRU
continue
resource_id = disk_specific_info.get("resource_id", "")
if disk_specific_info.get(resource_id):
assert (disk_specific_info.get(resource_id).get("ERROR")
is not None)
# Skip any validation on specific info if ERROR seen on sensor
continue
sensor_type = [
k if k.startswith("Sensor Type") else None
for k in disk_specific_info.keys()
][0]
assert (sensor_type is not None)
assert (disk_specific_info.get("resource_id") is not None)
if "States Asserted" in disk_specific_info:
assert (disk_specific_info.get("States Asserted") is not None)
else:
# Skip any validation if ERROR seen on the specifc FRU
if not disk_specific_infos.get("ERROR"):
sensor_type = [
k if k.startswith("Sensor Type") else None
for k in disk_specific_infos.keys()
][0]
assert (sensor_type is not None)
assert (disk_specific_infos.get("resource_id") is not None)
if "States Asserted" in disk_specific_infos:
assert (disk_specific_infos.get("States Asserted") is not None)
def test_real_stor_fan_module_actuator(agrs):
check_sspl_ll_is_running()
instance_id = "4"
resource_type = "enclosure:fru:fan"
send_enclosure_request("ENCL:%s" % resource_type, instance_id)
ingressMsg = get_fru_response(resource_type, instance_id)
fan_module_actuator_msg = ingressMsg.get("actuator_response_type")
assert (fan_module_actuator_msg is not None)
assert (fan_module_actuator_msg.get("alert_type") is not None)
assert (fan_module_actuator_msg.get("alert_id") is not None)
assert (fan_module_actuator_msg.get("severity") is not None)
assert (fan_module_actuator_msg.get("host_id") is not None)
assert (fan_module_actuator_msg.get("info") is not None)
fan_module_info = fan_module_actuator_msg.get("info")
assert (fan_module_info.get("site_id") is not None)
assert (fan_module_info.get("node_id") is not None)
assert (fan_module_info.get("cluster_id") is not None)
assert (fan_module_info.get("rack_id") is not None)
assert (fan_module_info.get("resource_type") is not None)
assert (fan_module_info.get("event_time") is not None)
assert (fan_module_info.get("resource_id") is not None)
fru_specific_info = fan_module_actuator_msg.get("specific_info", {})
resource_id = fan_module_info.get("resource_id")
if resource_id == "*":
verify_fan_module_specific_info(fru_specific_info)
return
if fru_specific_info:
assert (fru_specific_info.get("durable_id") is not None)
assert (fru_specific_info.get("status") is not None)
assert (fru_specific_info.get("name") is not None)
assert (fru_specific_info.get("enclosure_id") is not None)
assert (fru_specific_info.get("health") is not None)
assert (fru_specific_info.get("health_reason") is not None)
assert (fru_specific_info.get("location") is not None)
assert (fru_specific_info.get("health_recommendation") is not None)
assert (fru_specific_info.get("position") is not None)
fans = fan_module_actuator_msg.get("specific_info").get("fans", [])
if fans:
for fan in fans:
assert (fan.get("durable_id") is not None)
assert (fan.get("status") is not None)
assert (fan.get("name") is not None)
assert (fan.get("speed") is not None)
assert (fan.get("locator_led") is not None)
assert (fan.get("position") is not None)
assert (fan.get("location") is not None)
assert (fan.get("part_number") is not None)
assert (fan.get("serial_number") is not None)
assert (fan.get("fw_revision") is not None)
assert (fan.get("hw_revision") is not None)
assert (fan.get("health") is not None)
assert (fan.get("health_reason") is not None)
assert (fan.get("health_recommendation") is not None)
def test_bmc_interface(args):
check_sspl_ll_is_running()
# backup active bmc interface
BMC_IF_CONSUL_KEY, BMC_IF_CONSUL_VAL = backup_bmc_config()
if BMC_IF_CONSUL_VAL == "lan":
simulate_bmc_interface_alert.lan_channel_alert(BMC_IF_CONSUL_KEY,
BMC_IF_CONSUL_VAL)
else:
simulate_bmc_interface_alert.kcs_channel_alert(BMC_IF_CONSUL_KEY,
BMC_IF_CONSUL_VAL)
bmc_interface_message = None
for i in range(30):
if world.sspl_modules[
IngressProcessorTests.name()]._is_my_msgQ_empty():
time.sleep(2)
while not world.sspl_modules[
IngressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
IngressProcessorTests.name()]._read_my_msgQ()
print("Received: %s" % ingressMsg)
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("sensor_response_type")
if msg_type["info"]["resource_type"] == "node:bmc:interface:kcs" or \
msg_type["info"]["resource_type"] == "node:bmc:interface:rmcp":
bmc_interface_message = msg_type
break
except Exception as exception:
print(exception)
if bmc_interface_message:
break
time.sleep(1)
#restore bmc config and activate ipmisimtool
simulate_bmc_interface_alert.restore_config()
assert (bmc_interface_message is not None)
assert (bmc_interface_message.get("alert_type") is not None)
alert_type = bmc_interface_message.get("alert_type")
assert (alert_type == "fault")
assert (bmc_interface_message.get("alert_id") is not None)
assert (bmc_interface_message.get("severity") is not None)
assert (bmc_interface_message.get("host_id") is not None)
assert (bmc_interface_message.get("info") is not None)
bmc_interface_info = bmc_interface_message.get("info")
assert (bmc_interface_info.get("site_id") is not None)
assert (bmc_interface_info.get("rack_id") is not None)
assert (bmc_interface_info.get("node_id") is not None)
assert (bmc_interface_info.get("cluster_id") is not None)
assert (bmc_interface_info.get("resource_id") is not None)
assert (bmc_interface_info.get("description") is not None)
bmc_interface_specific_info = bmc_interface_message.get("specific_info")
if bmc_interface_specific_info:
assert (bmc_interface_specific_info.get("channel info") is not None)
def test_node_disk_module_actuator(agrs):
print("Enters into test_node_disk_module_actuator")
check_sspl_ll_is_running()
time.sleep(30)
instance_id = "*"
disk_actuator_message_request("NDHW:node:fru:disk", instance_id)
disk_actuator_msg = None
ingressMsg = {}
for i in range(10):
if world.sspl_modules[
IngressProcessorTests.name()]._is_my_msgQ_empty():
time.sleep(2)
while not world.sspl_modules[
IngressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
IngressProcessorTests.name()]._read_my_msgQ()
print("Received: %s " % ingressMsg)
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("actuator_response_type")
if msg_type["info"]["resource_type"] == "node:fru:disk" and \
msg_type["info"]["resource_id"] == instance_id:
# Break if condition is met
disk_actuator_msg = msg_type
break
except Exception as exception:
print(exception)
if disk_actuator_msg:
break
time.sleep(1)
assert (disk_actuator_msg is not None)
assert (disk_actuator_msg.get("alert_type") is not None)
# assert(disk_actuator_msg.get("alert_id") is not None)
assert (disk_actuator_msg.get("severity") is not None)
assert (disk_actuator_msg.get("host_id") is not None)
assert (disk_actuator_msg.get("info") is not None)
disk_actuator_info = disk_actuator_msg.get("info")
assert (disk_actuator_info.get("site_id") is not None)
assert (disk_actuator_info.get("node_id") is not None)
# assert(disk_actuator_info.get("cluster_id") is not None)
assert (disk_actuator_info.get("rack_id") is not None)
assert (disk_actuator_info.get("resource_type") is not None)
assert (disk_actuator_info.get("event_time") is not None)
assert (disk_actuator_info.get("resource_id") is not None)
disk_actuator_specific_infos = disk_actuator_msg.get("specific_info")
for disk_actuator_specific_info in disk_actuator_specific_infos:
assert (disk_actuator_specific_info is not None)
assert (disk_actuator_specific_info.get("Sensor Type (Discrete)")
is not None)
assert (disk_actuator_specific_info.get("resource_id") is not None)
if "States Asserted" in disk_actuator_specific_info:
assert (disk_actuator_specific_info.get("States Asserted")
is not None)
def test_service_restart_case(args):
check_sspl_ll_is_running()
check_service_is_running(service_name)
DbusServiceHandler().restart(service_name)
time.sleep(WAIT_TIME)
sensor_response = read_ingress_queue()
check_service_is_running(service_name)
if sensor_response:
print(sensor_response)
assert (sensor_response['info']['alert_type'] != "fault")
def test_service_failed_alert(args):
check_sspl_ll_is_running()
check_service_is_running(service_name)
simulate_service_alerts.simulate_fault_alert()
time.sleep(5)
sensor_response = read_ingress_queue()
assert_on_mismatch(sensor_response, "fault")
simulate_service_alerts.restore_service_file()
time.sleep(5)
sensor_response = read_ingress_queue()
assert_on_mismatch(sensor_response, "fault_resolved")
def test_node_psu_module_actuator(agrs):
check_sspl_ll_is_running()
instance_id = "*"
psu_actuator_message_request("NDHW:node:fru:psu", instance_id)
psu_module_actuator_msg = None
ingressMsg = {}
for i in range(10):
if world.sspl_modules[
IngressProcessorTests.name()]._is_my_msgQ_empty():
time.sleep(2)
while not world.sspl_modules[
IngressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
IngressProcessorTests.name()]._read_my_msgQ()
print("Received: %s " % ingressMsg)
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("actuator_response_type")
if msg_type["info"]["resource_type"] == "node:fru:psu" and \
msg_type["info"]["resource_id"] == instance_id:
psu_module_actuator_msg = msg_type
break
except Exception as exception:
print(exception)
if psu_module_actuator_msg:
break
time.sleep(1)
assert (ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID)
assert (psu_module_actuator_msg is not None)
assert (psu_module_actuator_msg.get("alert_type") is not None)
assert (psu_module_actuator_msg.get("severity") is not None)
assert (psu_module_actuator_msg.get("host_id") is not None)
assert (psu_module_actuator_msg.get("info") is not None)
assert (psu_module_actuator_msg.get("instance_id") is not None)
psu_module_info = psu_module_actuator_msg.get("info")
assert (psu_module_info.get("site_id") is not None)
assert (psu_module_info.get("node_id") is not None)
assert (psu_module_info.get("rack_id") is not None)
assert (psu_module_info.get("resource_type") is not None)
assert (psu_module_info.get("event_time") is not None)
assert (psu_module_info.get("resource_id") is not None)
fru_specific_infos = psu_module_actuator_msg.get("specific_info", {})
if fru_specific_infos:
for fru_specific_info in fru_specific_infos:
resource_id = fru_specific_info.get("resource_id")
assert (fru_specific_info.get("States Asserted") is not None)
assert (fru_specific_info.get("Sensor Type (Discrete)")
is not None)
assert (fru_specific_info.get("resource_id") is not None)
def test_cpu_data_sensor(args):
check_sspl_ll_is_running()
node_data_sensor_message_request("node:os:cpu_usage")
cpu_data_msg = None
sleep(10)
while not world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
RabbitMQingressProcessorTests.name()]._read_my_msgQ()
sleep(0.1)
print("Received for cpu_data: {0}".format(ingressMsg))
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("sensor_response_type")
if msg_type.get("info").get(
"resource_type") == "node:os:cpu_usage":
cpu_data_msg = msg_type
break
except Exception as exception:
sleep(0.1)
print(exception)
assert (cpu_data_msg is not None)
assert (cpu_data_msg.get("alert_type") is not None)
assert (cpu_data_msg.get("alert_id") is not None)
assert (cpu_data_msg.get("severity") is not None)
assert (cpu_data_msg.get("host_id") is not None)
assert (cpu_data_msg.get("info") is not None)
assert (cpu_data_msg.get("specific_info") is not None)
info = cpu_data_msg.get("info")
assert (info.get("site_id") is not None)
assert (info.get("node_id") is not None)
assert (info.get("cluster_id") is not None)
assert (info.get("rack_id") is not None)
assert (info.get("resource_type") is not None)
assert (info.get("event_time") is not None)
assert (info.get("resource_id") is not None)
assert (info.get("description") is not None)
specific_info = cpu_data_msg.get("specific_info")
assert (specific_info.get("systemTime") is not None)
assert (specific_info.get("interruptTime") is not None)
assert (specific_info.get("userTime") is not None)
assert (specific_info.get("idleTime") is not None)
assert (specific_info.get("csps") is not None)
assert (specific_info.get("iowaitTime") is not None)
assert (specific_info.get("niceTime") is not None)
assert (specific_info.get("cpu_usage") is not None)
assert (specific_info.get("coreData") is not None)
assert (specific_info.get("localtime") is not None)
assert (specific_info.get("softirqTime") is not None)
assert (specific_info.get("stealTime") is not None)
def test_service_inactive_alert(args):
check_sspl_ll_is_running()
check_service_is_running(service_name)
# Simulate Fault alert by stopping the service.
DbusServiceHandler().stop(service_name)
time.sleep(WAIT_TIME)
sensor_response = read_ingress_queue()
assert_on_mismatch(sensor_response, "fault")
# Simulate Fault resolved alert.
DbusServiceHandler().start(service_name)
time.sleep(5)
sensor_response = read_ingress_queue()
assert_on_mismatch(sensor_response, "fault_resolved")
def test_if_data_sensor(args):
check_sspl_ll_is_running()
node_data_sensor_message_request(resource_type)
if_data_msg = None
#create dummy interface to get network alerts
mock_eth_interface.shuffle_nw_interface()
# Wait untill expected resource type found in RMQ ingress processor msgQ.
start_time = time.time()
max_wait_time = 60
while not if_data_msg:
if not world.sspl_modules[
IngressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
IngressProcessorTests.name()]._read_my_msgQ()
print("Received: {0}".format(ingressMsg))
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("sensor_response_type")
if msg_type["info"]["resource_type"] == resource_type:
if_data_msg = msg_type
except Exception as exception:
print(exception)
if (time.time() - start_time) > max_wait_time:
break
assert (if_data_msg is not None)
assert (if_data_msg.get("alert_type") is not None)
assert (if_data_msg.get("alert_id") is not None)
assert (if_data_msg.get("severity") is not None)
assert (if_data_msg.get("host_id") is not None)
assert (if_data_msg.get("info") is not None)
assert (if_data_msg.get("specific_info") is not None)
if_data_info = if_data_msg.get("info")
assert (if_data_info.get("site_id") is not None)
assert (if_data_info.get("node_id") is not None)
assert (if_data_info.get("cluster_id") is not None)
assert (if_data_info.get("rack_id") is not None)
assert (if_data_info.get("resource_type") == resource_type)
assert (if_data_info.get("event_time") is not None)
assert (if_data_info.get("resource_id") is not None)
assert (if_data_info.get("description") is not None)
if_data_specific_info = if_data_msg.get("specific_info")
assert (if_data_specific_info is not None)
assert (if_data_specific_info.get("localtime") is not None)
assert (if_data_specific_info.get("interfaces") is not None)
# Remove interface added for test purpose
mock_eth_interface.delete_nw_interface()
def test_iem_alerts(self):
"""Test iem 'ipmitool' fault alert receive."""
check_sspl_ll_is_running()
Iem().iem_fault("IPMITOOL_ERROR")
time.sleep(10)
EventMessage.subscribe(component='sspl')
fault_alert = EventMessage.receive()
print(f"IEM Received:{fault_alert}")
assert (fault_alert is not None)
assert (fault_alert["iem"]["info"]["severity"] is not None)
assert (fault_alert["iem"]["info"]["type"] is not None)
assert (fault_alert["iem"]["info"]["event_time"] is not None)
assert (fault_alert["iem"]["source"]["module"] is not None)
assert (fault_alert["iem"]["contents"]["event"] is not None)
def test_real_stor_enclosure_sensor(agrs):
timeout = time.time() + 60 * 3
check_sspl_ll_is_running()
kill_mock_server()
encl_sensor_message_request("enclosure")
encl_sensor_msg = None
while time.time() < timeout:
if world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
time.sleep(1)
continue
ingressMsg = world.sspl_modules[
RabbitMQingressProcessorTests.name()]._read_my_msgQ()
time.sleep(0.1)
print("Received: %s" % ingressMsg)
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("sensor_response_type")
if msg_type["info"]["resource_type"] == "enclosure":
encl_sensor_msg = ingressMsg.get("sensor_response_type")
break
except Exception as exception:
time.sleep(0.1)
print(exception)
if encl_sensor_msg:
break
assert (encl_sensor_msg is not None
), "Timeout error, Real Store Enclosure Sensor test is failed."
assert (encl_sensor_msg.get("alert_type") is not None)
assert (encl_sensor_msg.get("alert_id") is not None)
assert (encl_sensor_msg.get("severity") is not None)
assert (encl_sensor_msg.get("host_id") is not None)
assert (encl_sensor_msg.get("info") is not None)
encl_sensor_info = encl_sensor_msg.get("info")
assert (encl_sensor_info.get("site_id") is not None)
assert (encl_sensor_info.get("rack_id") is not None)
assert (encl_sensor_info.get("node_id") is not None)
assert (encl_sensor_info.get("cluster_id") is not None)
assert (encl_sensor_info.get("resource_id") is not None)
assert (encl_sensor_info.get("resource_type") is not None)
assert (encl_sensor_info.get("event_time") is not None)
assert (encl_sensor_info.get("description") is not None)
encl_specific_info = encl_sensor_msg.get("specific_info")
if encl_specific_info:
assert (encl_specific_info.get("event") is not None)
def test_host_update_data_sensor(args):
check_sspl_ll_is_running()
node_data_sensor_message_request("node:os:memory_usage")
host_update_msg = None
sleep(10)
while not world.sspl_modules[
IngressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
IngressProcessorTests.name()]._read_my_msgQ()
sleep(0.1)
print("Received for host_data: {0}".format(ingressMsg))
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("sensor_response_type")
if msg_type.get("info").get(
"resource_type") == "node:os:memory_usage":
host_update_msg = msg_type
break
except Exception as exception:
sleep(0.1)
print(exception)
assert (host_update_msg is not None)
assert (host_update_msg.get("alert_type") is not None)
assert (host_update_msg.get("alert_id") is not None)
assert (host_update_msg.get("severity") is not None)
assert (host_update_msg.get("host_id") is not None)
assert (host_update_msg.get("info") is not None)
assert (host_update_msg.get("specific_info") is not None)
info = host_update_msg.get("info")
assert (info.get("site_id") is not None)
assert (info.get("node_id") is not None)
assert (info.get("cluster_id") is not None)
assert (info.get("rack_id") is not None)
assert (info.get("resource_type") is not None)
assert (info.get("event_time") is not None)
assert (info.get("resource_id") is not None)
assert (info.get("description") is not None)
specific_info = host_update_msg.get("specific_info")
assert (specific_info.get("loggedInUsers") is not None)
assert (specific_info.get("totalMemory") is not None)
assert (specific_info.get("runningProcessCount") is not None)
assert (specific_info.get("uname") is not None)
assert (specific_info.get("bootTime") is not None)
assert (specific_info.get("processCount") is not None)
assert (specific_info.get("localtime") is not None)
def test_systemd_service_invalid_request(args):
service_name = "temp_dummy.service"
request = "start"
check_sspl_ll_is_running()
service_actuator_request(service_name, request)
service_actuator_msg = None
ingressMsg = {}
for i in range(10):
if world.sspl_modules[
IngressProcessorTests.name()]._is_my_msgQ_empty():
time.sleep(2)
while not world.sspl_modules[IngressProcessorTests.name()]\
._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[IngressProcessorTests.name()]\
._read_my_msgQ()
print("Received: %s " % ingressMsg)
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("actuator_response_type")
if msg_type["info"]["resource_type"] == RESOURCE_TYPE and \
msg_type["info"]["resource_id"] == service_name:
# Break if required condition met
service_actuator_msg = msg_type
break
except Exception as exception:
print(exception)
if service_actuator_msg:
break
time.sleep(1)
assert (service_actuator_msg is not None)
assert (service_actuator_msg.get("alert_type") == "UPDATE")
assert (service_actuator_msg.get("severity") is not None)
assert (service_actuator_msg.get("host_id") is not None)
assert (service_actuator_msg.get("info") is not None)
info = service_actuator_msg.get("info")
assert (info.get("site_id") is not None)
assert (info.get("node_id") is not None)
assert (info.get("cluster_id") is not None)
assert (info.get("rack_id") is not None)
assert (info.get("resource_type") == RESOURCE_TYPE)
assert (info.get("event_time") is not None)
assert (info.get("resource_id") == service_name)
assert (service_actuator_msg.get("specific_info") is not None)
specific_info = service_actuator_msg.get("specific_info")
assert (specific_info[0].get("error_msg") is not None)
def test_raid_integrity_sensor(args):
platform = check_os_platform()
if platform == "vm":
return
check_sspl_ll_is_running()
node_data_sensor_message_request("node:os:raid_integrity")
raid_data_msg = None
sleep(10)
while not world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
RabbitMQingressProcessorTests.name()]._read_my_msgQ()
sleep(0.1)
print("Received for raid_integrity_data: {0}".format(ingressMsg))
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("sensor_response_type")
if msg_type.get("info").get(
"resource_type") == "node:os:raid_integrity":
raid_data_msg = msg_type
break
except Exception as exception:
sleep(0.1)
print(exception)
assert (raid_data_msg is not None)
assert (raid_data_msg.get("alert_type") is not None)
assert (raid_data_msg.get("alert_id") is not None)
assert (raid_data_msg.get("severity") is not None)
assert (raid_data_msg.get("host_id") is not None)
assert (raid_data_msg.get("info") is not None)
assert (raid_data_msg.get("specific_info") is not None)
info = raid_data_msg.get("info")
assert (info.get("site_id") is not None)
assert (info.get("node_id") is not None)
assert (info.get("cluster_id") is not None)
assert (info.get("rack_id") is not None)
assert (info.get("resource_type") is not None)
assert (info.get("event_time") is not None)
assert (info.get("resource_id") is not None)
assert (info.get("description") is not None)
specific_info = raid_data_msg.get("specific_info")
assert (specific_info.get("error") is None)
def test_node_voltage_sensor(agrs):
check_sspl_ll_is_running()
instance_id = "*"
resource_type = "node:sensor:voltage"
send_node_controller_message_request(UUID, "NDHW:%s" % resource_type, instance_id)
ingressMsg = get_fru_response(resource_type, instance_id)
assert(ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID)
sensor_msg = ingressMsg.get("actuator_response_type")
assert(sensor_msg is not None)
assert(sensor_msg.get("alert_type") is not None)
assert(sensor_msg.get("severity") is not None)
assert(sensor_msg.get("host_id") is not None)
assert(sensor_msg.get("info") is not None)
assert(sensor_msg.get("instance_id") == instance_id)
volt_module_info = sensor_msg.get("info")
assert(volt_module_info.get("site_id") is not None)
assert(volt_module_info.get("node_id") is not None)
assert(volt_module_info.get("rack_id") is not None)
assert(volt_module_info.get("resource_type") is not None)
assert(volt_module_info.get("event_time") is not None)
assert(volt_module_info.get("resource_id") is not None)
fru_specific_infos = sensor_msg.get("specific_info")
assert(fru_specific_infos is not None)
for fru_specific_info in fru_specific_infos:
assert(fru_specific_info is not None)
if fru_specific_info.get("ERROR"):
# Skip any validation on specific info if ERROR seen on FRU
continue
assert(fru_specific_info.get("resource_id") is not None)
resource_id = fru_specific_info.get("resource_id")
if fru_specific_info.get(resource_id):
assert(fru_specific_info.get(resource_id).get("ERROR") is not None)
# Skip any validation on specific info if ERROR seen on sensor
continue
assert(fru_specific_info.get("sensor_number") is not None)
assert(fru_specific_info.get("sensor_status") is not None)
assert(fru_specific_info.get("entity_id_instance") is not None)
assert(fru_specific_info.get("sensor_reading") is not None)
def test_real_stor_dg_sensor(agrs):
check_sspl_ll_is_running()
dg_sensor_message_request("enclosure:cortx:disk_group")
ingress_msg_type = "sensor_response_type"
ingress_msg = get_fru_response("enclosure:cortx:disk_group", None,
ingress_msg_type)
dg_sensor_msg = ingress_msg.get(ingress_msg_type)
assert (dg_sensor_msg is not None)
assert (dg_sensor_msg.get("alert_type") is not None)
assert (dg_sensor_msg.get("alert_id") is not None)
assert (dg_sensor_msg.get("severity") is not None)
assert (dg_sensor_msg.get("host_id") is not None)
assert (dg_sensor_msg.get("info") is not None)
dg_sensor_info = dg_sensor_msg.get("info")
assert (dg_sensor_info.get("resource_type") is not None)
assert (dg_sensor_info.get("event_time") is not None)
assert (dg_sensor_info.get("resource_id") is not None)
dg_sensor_specific_info = dg_sensor_msg.get("specific_info")
assert (dg_sensor_specific_info is not None)
assert (dg_sensor_specific_info.get("cvg_name") == 'A01')
assert (dg_sensor_specific_info.get("cvg_id") == 0)
assert (dg_sensor_specific_info.get("object_name") is not None)
assert (dg_sensor_specific_info.get("name") is not None)
assert (dg_sensor_specific_info.get("size") is not None)
assert (dg_sensor_specific_info.get("freespace") is not None)
assert (dg_sensor_specific_info.get("storage_type") is not None)
assert (dg_sensor_specific_info.get("pool") is not None)
assert (dg_sensor_specific_info.get("pool_serial_number") is not None)
assert (dg_sensor_specific_info.get("pool_percentage") is not None)
assert (dg_sensor_specific_info.get("owner") is not None)
assert (dg_sensor_specific_info.get("raidtype") is not None)
assert (dg_sensor_specific_info.get("status") is not None)
assert (dg_sensor_specific_info.get("create_date") is not None)
assert (dg_sensor_specific_info.get("disk_description") is not None)
assert (dg_sensor_specific_info.get("serial_number") is not None)
assert (dg_sensor_specific_info.get("pool_sector_format") is not None)
assert (dg_sensor_specific_info.get("health") is not None)
assert (dg_sensor_specific_info.get("health_reason") is not None)
assert (dg_sensor_specific_info.get("health_recommendation") is not None)
def test_cpu_data_sensor(args):
resource_type = "node:os:cpu_usage"
instance_id = "*"
ingress_msg_type="sensor_response_type"
check_sspl_ll_is_running()
node_data_sensor_message_request(resource_type)
ingress_msg = get_fru_response(
resource_type, instance_id, ingress_msg_type)
cpu_data_msg = ingress_msg.get(ingress_msg_type)
assert(cpu_data_msg is not None)
assert(cpu_data_msg.get("alert_type") is not None)
assert(cpu_data_msg.get("alert_id") is not None)
assert(cpu_data_msg.get("severity") is not None)
assert(cpu_data_msg.get("host_id") is not None)
assert(cpu_data_msg.get("info") is not None)
assert(cpu_data_msg.get("specific_info") is not None)
info = cpu_data_msg.get("info")
assert(info.get("site_id") is not None)
assert(info.get("node_id") is not None)
assert(info.get("cluster_id") is not None)
assert(info.get("rack_id") is not None)
assert(info.get("resource_type") is not None)
assert(info.get("event_time") is not None)
assert(info.get("resource_id") is not None)
assert(info.get("description") is not None)
specific_info = cpu_data_msg.get("specific_info")
assert(specific_info.get("systemTime") is not None)
assert(specific_info.get("interruptTime") is not None)
assert(specific_info.get("userTime") is not None)
assert(specific_info.get("idleTime") is not None)
assert(specific_info.get("csps") is not None)
assert(specific_info.get("iowaitTime") is not None)
assert(specific_info.get("niceTime") is not None)
assert(specific_info.get("cpu_usage") is not None)
assert(specific_info.get("coreData") is not None)
assert(specific_info.get("localtime") is not None)
assert(specific_info.get("softirqTime") is not None)
assert(specific_info.get("stealTime") is not None)
def test_systemd_service_valid_request(args):
service_name = "rsyslog.service"
request = "status"
check_sspl_ll_is_running()
# TODO: Change service name, once get final 3rd party service name
service_actuator_request(service_name, request)
service_actuator_msg = None
time.sleep(8)
ingressMsg = {}
while not world.sspl_modules[IngressProcessorTests.name()]\
._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[IngressProcessorTests.name()]\
._read_my_msgQ()
time.sleep(0.1)
print("Received: %s " % ingressMsg)
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("actuator_response_type")
if msg_type["info"]["resource_type"] == RESOURCE_TYPE:
service_actuator_msg = msg_type
break
except Exception as exception:
time.sleep(0.1)
print(exception)
assert(service_actuator_msg is not None)
assert(service_actuator_msg.get("alert_type") == "UPDATE")
assert(service_actuator_msg.get("severity") is not None)
assert(service_actuator_msg.get("host_id") is not None)
assert(service_actuator_msg.get("info") is not None)
info = service_actuator_msg.get("info")
assert(info.get("site_id") is not None)
assert(info.get("node_id") is not None)
assert(info.get("cluster_id") is not None)
assert(info.get("rack_id") is not None)
assert(info.get("resource_type") == RESOURCE_TYPE)
assert(info.get("event_time") is not None)
assert(info.get("resource_id") == service_name)
assert(service_actuator_msg.get("specific_info") is not None)
def test_self_hw_node_sel_event(args):
check_sspl_ll_is_running()
# fetch a good resource fron ipmitool
result = run_cmd('ipmitool sdr type Fan')
test_resource = None
if result:
for resource in result:
if 'ok' in resource.decode().lower():
# this is the first ok resource, use it
test_resource = resource.decode().split('|')[0].strip()
break
# inject event into sel list and wait for alert
print(f"Using test resource {test_resource}")
run_cmd(f"ipmitool event '{test_resource}' lcr")
# wait for fault alert
start_time = time.time()
asserted = False
while time.time() - start_time < 60: # wait for 60 seconds
if wait_for_asserted_event():
asserted = True
break
if not asserted:
print("Did not get asserted event alert.")
assert (False)
# revert the event
run_cmd(f"ipmitool event '{test_resource}' lcr deassert")
# wait for alert
start_time = time.time()
deasserted = False
while time.time() - start_time < 60: # wait for 60 seconds
if wait_for_deasserted_event():
deasserted = True
break
if not deasserted:
print("Did not get asserted event alert.")
assert (False)
else:
print("ipmitool returned no results.")
assert (False)
def test_host_update_data_sensor(args):
resource_type = "node:os:memory_usage"
instance_id = "*"
ingress_msg_type="sensor_response_type"
check_sspl_ll_is_running()
node_data_sensor_message_request(resource_type)
ingress_msg = get_fru_response(
resource_type, instance_id, ingress_msg_type)
host_update_msg = ingress_msg.get(ingress_msg_type)
assert(host_update_msg is not None)
assert(host_update_msg.get("alert_type") is not None)
assert(host_update_msg.get("alert_id") is not None)
assert(host_update_msg.get("severity") is not None)
assert(host_update_msg.get("host_id") is not None)
assert(host_update_msg.get("info") is not None)
assert(host_update_msg.get("specific_info") is not None)
info = host_update_msg.get("info")
assert(info.get("site_id") is not None)
assert(info.get("node_id") is not None)
assert(info.get("cluster_id") is not None)
assert(info.get("rack_id") is not None)
assert(info.get("resource_type") is not None)
assert(info.get("event_time") is not None)
assert(info.get("resource_id") is not None)
assert(info.get("description") is not None)
specific_info = host_update_msg.get("specific_info")
assert(specific_info.get("loggedInUsers") is not None)
assert(specific_info.get("totalMemory") is not None)
assert(specific_info.get("runningProcessCount") is not None)
assert(specific_info.get("uname") is not None)
assert(specific_info.get("bootTime") is not None)
assert(specific_info.get("processCount") is not None)
assert(specific_info.get("localtime") is not None)
def test_real_stor_sideplane_module_actuator(agrs):
check_sspl_ll_is_running()
# sideplane_actuator_message_request("ENCL:enclosure:fru:sideplane", "Left Sideplane")
sideplane_actuator_message_request("ENCL:enclosure:fru:sideplane", "*")
sideplane_module_actuator_msg = None
time.sleep(4)
while not world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
RabbitMQingressProcessorTests.name()]._read_my_msgQ()
time.sleep(0.1)
print("Received: %s" % ingressMsg)
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("actuator_response_type")
if msg_type["info"]["resource_type"] == "enclosure:fru:sideplane":
sideplane_module_actuator_msg = msg_type
break
except Exception as exception:
time.sleep(0.1)
print(exception)
assert (sideplane_module_actuator_msg is not None)
assert (sideplane_module_actuator_msg.get("alert_type") is not None)
assert (sideplane_module_actuator_msg.get("alert_id") is not None)
assert (sideplane_module_actuator_msg.get("severity") is not None)
assert (sideplane_module_actuator_msg.get("host_id") is not None)
assert (sideplane_module_actuator_msg.get("info") is not None)
sideplane_module_info = sideplane_module_actuator_msg.get("info")
assert (sideplane_module_info.get("site_id") is not None)
assert (sideplane_module_info.get("node_id") is not None)
assert (sideplane_module_info.get("cluster_id") is not None)
assert (sideplane_module_info.get("rack_id") is not None)
assert (sideplane_module_info.get("resource_type") is not None)
assert (sideplane_module_info.get("event_time") is not None)
assert (sideplane_module_info.get("resource_id") is not None)
sideplane_specific_info = sideplane_module_actuator_msg.get(
"specific_info", {})
resource_id = sideplane_module_info.get("resource_id")
if resource_id == "*":
verify_sideplane_module_specific_info(sideplane_specific_info)
return
if sideplane_specific_info:
assert (sideplane_specific_info.get("object_name") is not None)
assert (sideplane_specific_info.get("durable_id") is not None)
assert (sideplane_specific_info.get("status") is not None)
assert (sideplane_specific_info.get("name") is not None)
assert (sideplane_specific_info.get("enclosure_id") is not None)
assert (sideplane_specific_info.get("drawer_id") is not None)
assert (sideplane_specific_info.get("dom_id") is not None)
assert (sideplane_specific_info.get("path_id") is not None)
assert (sideplane_specific_info.get("path_id_numeric") is not None)
assert (sideplane_specific_info.get("location") is not None)
assert (sideplane_specific_info.get("position") is not None)
assert (sideplane_specific_info.get("position_numeric") is not None)
assert (sideplane_specific_info.get("status_numeric") is not None)
assert (sideplane_specific_info.get("extended_status") is not None)
assert (sideplane_specific_info.get("health") is not None)
assert (sideplane_specific_info.get("health_numeric") is not None)
assert (sideplane_specific_info.get("health_reason") is not None)
assert (sideplane_specific_info.get("health_recommendation")
is not None)
expanders = sideplane_module_actuator_msg.get("specific_info").get(
"sideplanes", [])
if expanders:
for expander in expanders:
assert (expander.get("object_name") is not None)
assert (expander.get("durable_id") is not None)
assert (expander.get("status") is not None)
assert (expander.get("name") is not None)
assert (expander.get("enclosure_id") is not None)
assert (expander.get("drawer_id") is not None)
assert (expander.get("dom_id") is not None)
assert (expander.get("path_id") is not None)
assert (expander.get("path_id_numeric") is not None)
assert (expander.get("location") is not None)
assert (expander.get("status_numeric") is not None)
assert (expander.get("extended_status") is not None)
assert (expander.get("fw_revision") is not None)
assert (expander.get("health") is not None)
assert (expander.get("health_numeric") is not None)
assert (expander.get("health_reason") is not None)
assert (expander.get("health_recommendation") is not None)
def test_node_fan_module_actuator(agrs):
check_sspl_ll_is_running()
instance_id = "*"
resource_type = "node:fru:fan"
send_node_controller_message_request(UUID, "NDHW:%s" % resource_type,
instance_id)
ingressMsg = get_fru_response(resource_type, instance_id)
assert (ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID)
fan_module_actuator_msg = ingressMsg.get("actuator_response_type")
assert (fan_module_actuator_msg is not None)
assert (fan_module_actuator_msg.get("alert_type") is not None)
assert (fan_module_actuator_msg.get("severity") is not None)
assert (fan_module_actuator_msg.get("host_id") is not None)
assert (fan_module_actuator_msg.get("info") is not None)
assert (fan_module_actuator_msg.get("instance_id") == instance_id)
fan_module_info = fan_module_actuator_msg.get("info")
assert (fan_module_info.get("site_id") is not None)
assert (fan_module_info.get("node_id") is not None)
assert (fan_module_info.get("rack_id") is not None)
assert (fan_module_info.get("resource_type") is not None)
assert (fan_module_info.get("event_time") is not None)
assert (fan_module_info.get("resource_id") is not None)
fan_specific_infos = fan_module_actuator_msg.get("specific_info")
assert (fan_specific_infos is not None)
if fan_module_info.get("resource_id") == "*":
for fan_specific_info in fan_specific_infos:
assert (fan_specific_info is not None)
if fan_specific_info.get("ERROR"):
# Skip any validation on specific info if ERROR seen on FRU
continue
resource_id = fan_specific_info.get("resource_id", "")
if fan_specific_info.get(resource_id):
assert (fan_specific_info.get(resource_id).get("ERROR")
is not None)
# Skip any validation on specific info if ERROR seen on sensor
continue
if "Fan Fail" in resource_id or "Fan Redundancy" in resource_id:
sensor_type = [
k if k.startswith("Sensor Type") else None
for k in fan_specific_info.keys()
][0]
assert (sensor_type is not None)
assert (fan_specific_info.get("resource_id") is not None)
assert (fan_specific_info.get("States Asserted") is not None)
else:
assert (fan_specific_info.get("resource_id") is not None)
sensor_type = [
k if k.startswith("Sensor Type") else None
for k in fan_specific_info.keys()
][0]
assert (sensor_type is not None)
assert (fan_specific_info.get("Sensor Reading") is not None)
assert (fan_specific_info.get("Status") is not None)
assert (fan_specific_info.get("Lower Non_Recoverable")
is not None)
assert (fan_specific_info.get("Lower Critical") is not None)
assert (fan_specific_info.get("Lower Non_Critical")
is not None)
assert (fan_specific_info.get("Upper Non_Critical")
is not None)
assert (fan_specific_info.get("Upper Critical") is not None)
assert (fan_specific_info.get("Upper Non_Recoverable")
is not None)
assert (fan_specific_info.get("Positive Hysteresis")
is not None)
assert (fan_specific_info.get("Negative Hysteresis")
is not None)
assert (fan_specific_info.get("Assertion Events") is not None)
assert (fan_specific_info.get("Assertions Enabled")
is not None)
assert (fan_specific_info.get("Deassertions Enabled")
is not None)
else:
# Skip any validation if ERROR seen on the specifc FRU
if not fan_specific_infos.get("ERROR"):
sensor_type = [
k if k.startswith("Sensor Type") else None
for k in fan_specific_infos.keys()
][0]
assert (sensor_type is not None)
assert (fan_specific_infos.get("Sensor Reading") is not None)
assert (fan_specific_infos.get("Status") is not None)
assert (fan_specific_infos.get("Lower Non_Recoverable")
is not None)
assert (fan_specific_infos.get("Lower Critical") is not None)
assert (fan_specific_infos.get("Lower Non_Critical") is not None)
assert (fan_specific_infos.get("Upper Non_Critical") is not None)
assert (fan_specific_infos.get("Upper Critical") is not None)
assert (fan_specific_infos.get("Upper Non_Recoverable")
is not None)
assert (fan_specific_infos.get("Positive Hysteresis") is not None)
assert (fan_specific_infos.get("Negative Hysteresis") is not None)
assert (fan_specific_infos.get("Assertion Events") is not None)
assert (fan_specific_infos.get("Assertions Enabled") is not None)
assert (fan_specific_infos.get("Deassertions Enabled") is not None)
assert (fan_specific_infos.get("resource_id") is not None)
def test_real_stor_sideplane_module_actuator(agrs):
check_sspl_ll_is_running()
instance_id = "*"
resource_type = "enclosure:hw:sideplane"
send_enclosure_request("ENCL:%s" % resource_type, instance_id)
ingressMsg = get_fru_response(resource_type, instance_id)
sideplane_module_actuator_msg = ingressMsg.get("actuator_response_type")
assert (sideplane_module_actuator_msg is not None)
assert (sideplane_module_actuator_msg.get("alert_type") is not None)
assert (sideplane_module_actuator_msg.get("alert_id") is not None)
assert (sideplane_module_actuator_msg.get("severity") is not None)
assert (sideplane_module_actuator_msg.get("host_id") is not None)
assert (sideplane_module_actuator_msg.get("info") is not None)
sideplane_module_info = sideplane_module_actuator_msg.get("info")
assert (sideplane_module_info.get("site_id") is not None)
assert (sideplane_module_info.get("node_id") is not None)
assert (sideplane_module_info.get("cluster_id") is not None)
assert (sideplane_module_info.get("rack_id") is not None)
assert (sideplane_module_info.get("resource_type") is not None)
assert (sideplane_module_info.get("event_time") is not None)
assert (sideplane_module_info.get("resource_id") is not None)
sideplane_specific_info = sideplane_module_actuator_msg.get(
"specific_info", {})
resource_id = sideplane_module_info.get("resource_id")
if resource_id == "*":
verify_sideplane_module_specific_info(sideplane_specific_info)
return
if sideplane_specific_info:
assert (sideplane_specific_info.get("object_name") is not None)
assert (sideplane_specific_info.get("durable_id") is not None)
assert (sideplane_specific_info.get("status") is not None)
assert (sideplane_specific_info.get("name") is not None)
assert (sideplane_specific_info.get("enclosure_id") is not None)
assert (sideplane_specific_info.get("drawer_id") is not None)
assert (sideplane_specific_info.get("dom_id") is not None)
assert (sideplane_specific_info.get("path_id") is not None)
assert (sideplane_specific_info.get("path_id_numeric") is not None)
assert (sideplane_specific_info.get("location") is not None)
assert (sideplane_specific_info.get("position") is not None)
assert (sideplane_specific_info.get("position_numeric") is not None)
assert (sideplane_specific_info.get("status_numeric") is not None)
assert (sideplane_specific_info.get("extended_status") is not None)
assert (sideplane_specific_info.get("health") is not None)
assert (sideplane_specific_info.get("health_numeric") is not None)
assert (sideplane_specific_info.get("health_reason") is not None)
assert (sideplane_specific_info.get("health_recommendation")
is not None)
expanders = sideplane_module_actuator_msg.get("specific_info").get(
"sideplanes", [])
if expanders:
for expander in expanders:
assert (expander.get("object_name") is not None)
assert (expander.get("durable_id") is not None)
assert (expander.get("status") is not None)
assert (expander.get("name") is not None)
assert (expander.get("enclosure_id") is not None)
assert (expander.get("drawer_id") is not None)
assert (expander.get("dom_id") is not None)
assert (expander.get("path_id") is not None)
assert (expander.get("path_id_numeric") is not None)
assert (expander.get("location") is not None)
assert (expander.get("status_numeric") is not None)
assert (expander.get("extended_status") is not None)
assert (expander.get("fw_revision") is not None)
assert (expander.get("health") is not None)
assert (expander.get("health_numeric") is not None)
assert (expander.get("health_reason") is not None)
assert (expander.get("health_recommendation") is not None)
def test_node_fan_module_actuator(agrs):
check_sspl_ll_is_running()
fan_actuator_message_request("NDHW:node:fru:fan", str(test_resource))
fan_module_actuator_msg = None
ingressMsg = {}
for i in range(10):
if world.sspl_modules[IngressProcessorTests.name()]._is_my_msgQ_empty():
time.sleep(2)
while not world.sspl_modules[IngressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[IngressProcessorTests.name()]._read_my_msgQ()
time.sleep(0.1)
print("Received: %s" % ingressMsg)
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("actuator_response_type")
if msg_type["info"]["resource_type"] == "node:fru:fan" and \
msg_type["info"]["resource_id"] == test_resource:
# Break if condition is met
fan_module_actuator_msg = msg_type
break
except Exception as exception:
time.sleep(0.1)
print(exception)
if fan_module_actuator_msg:
break
time.sleep(1)
assert(ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID)
assert(fan_module_actuator_msg is not None)
assert(fan_module_actuator_msg.get("alert_type") is not None)
assert(fan_module_actuator_msg.get("severity") is not None)
assert(fan_module_actuator_msg.get("host_id") is not None)
assert(fan_module_actuator_msg.get("info") is not None)
assert(fan_module_actuator_msg.get("instance_id") is not None)
fan_module_info = fan_module_actuator_msg.get("info")
assert(fan_module_info.get("site_id") is not None)
assert(fan_module_info.get("node_id") is not None)
assert(fan_module_info.get("rack_id") is not None)
assert(fan_module_info.get("resource_type") is not None)
assert(fan_module_info.get("event_time") is not None)
assert(fan_module_info.get("resource_id") is not None)
fru_specific_infos = fan_module_actuator_msg.get("specific_info", {})
if fru_specific_infos and fan_module_info.get("resource_id") == "*":
for fru_specific_info in fru_specific_infos:
resource_id = fru_specific_info.get("resource_id")
if "Fan Fail" in resource_id:
assert(fru_specific_info.get("Sensor Type (Discrete)") is not None)
assert(fru_specific_info.get("resource_id") is not None)
elif "System Fan" in resource_id:
assert(fru_specific_info.get("Status") is not None)
assert(fru_specific_info.get("Sensor Type (Threshold)") is not None)
assert(fru_specific_info.get("Sensor Reading") is not None)
assert(fru_specific_info.get("Lower Non-Recoverable") is not None)
assert(fru_specific_info.get("Assertions Enabled") is not None)
assert(fru_specific_info.get("Upper Non-Critical") is not None)
assert(fru_specific_info.get("Upper Non-Recoverable") is not None)
assert(fru_specific_info.get("Positive Hysteresis") is not None)
assert(fru_specific_info.get("Lower Critical") is not None)
assert(fru_specific_info.get("Deassertions Enabled") is not None)
assert(fru_specific_info.get("Lower Non-Critical") is not None)
assert(fru_specific_info.get("Upper Critical") is not None)
assert(fru_specific_info.get("Negative Hysteresis") is not None)
assert(fru_specific_info.get("Assertion Events") is not None)
assert(fru_specific_info.get("resource_id") is not None)
else:
assert(fru_specific_info.get("States Asserted") is not None)
assert(fru_specific_info.get("Sensor Type (Discrete)") is not None)
assert(fru_specific_info.get("resource_id") is not None)
elif fru_specific_infos:
assert(fru_specific_infos.get("Sensor Type (Threshold)") is not None)
assert(fru_specific_infos.get("Sensor Reading") is not None)
assert(fru_specific_infos.get("Status") is not None)
assert(fru_specific_infos.get("Lower Non_Recoverable") is not None)
assert(fru_specific_infos.get("Lower Critical") is not None)
assert(fru_specific_infos.get("Lower Non_Critical") is not None)
assert(fru_specific_infos.get("Upper Non_Critical") is not None)
assert(fru_specific_infos.get("Upper Critical") is not None)
assert(fru_specific_infos.get("Upper Non_Recoverable") is not None)
assert(fru_specific_infos.get("Positive Hysteresis") is not None)
assert(fru_specific_infos.get("Negative Hysteresis") is not None)
assert(fru_specific_infos.get("Assertion Events") is not None)
assert(fru_specific_infos.get("Assertions Enabled") is not None)
assert(fru_specific_infos.get("Deassertions Enabled") is not None)
assert(fru_specific_infos.get("resource_id") is not None)
