def test_sensor_unrecoverable_failure_alert(args): """ Check for fault alert on unrecoverable sensor module failure. Simulate sensor failure Verify fault alert on unrecoverable sensor module Verify sensor module is not running """ check_sspl_ll_is_running() simulate_NodeHWsensor_failure() # Wait until know its unrecoverable error ingressMsg = get_fru_response( resource_type, resource_id, ingress_msg_type="sensor_response_type", timeout=get_maximum_recovery_time(resource_id), alert_type="fault") sensor_msg = ingressMsg.get("sensor_response_type") assert sensor_msg assert sensor_msg.get("alert_type") == "fault" assert sensor_msg.get("severity") == "critical" assert sensor_msg.get("host_id") info = sensor_msg.get("info") assert info assert info.get("site_id") assert info.get("node_id") assert info.get("rack_id") assert info.get("event_time") assert info.get("impact") assert info.get("recommendation") == "Restart SSPL service" assert info.get("description") assert info.get("resource_type") == resource_type assert info.get("resource_id") == resource_id execute_module_thread_operation(action="status", thread_response="Status: Halted")
def test_node_disk_module_actuator(agrs): check_sspl_ll_is_running() instance_id = "*" resource_type = "node:fru:disk" send_node_controller_message_request(UUID, "NDHW:%s" % resource_type, instance_id) ingressMsg = get_fru_response(resource_type, instance_id) assert (ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID) disk_actuator_msg = ingressMsg.get("actuator_response_type") assert (disk_actuator_msg is not None) assert (disk_actuator_msg.get("alert_type") is not None) assert (disk_actuator_msg.get("severity") is not None) assert (disk_actuator_msg.get("host_id") is not None) assert (disk_actuator_msg.get("info") is not None) assert (disk_actuator_msg.get("instance_id") == instance_id) disk_actuator_info = disk_actuator_msg.get("info") assert (disk_actuator_info.get("site_id") is not None) assert (disk_actuator_info.get("node_id") is not None) assert (disk_actuator_info.get("rack_id") is not None) assert (disk_actuator_info.get("resource_type") is not None) assert (disk_actuator_info.get("event_time") is not None) assert (disk_actuator_info.get("resource_id") is not None) disk_specific_infos = disk_actuator_msg.get("specific_info") assert (disk_specific_infos is not None) if disk_actuator_info.get("resource_id") == "*": for disk_specific_info in disk_specific_infos: assert (disk_specific_info is not None) if disk_specific_info.get("ERROR"): # Skip any validation on specific info if ERROR seen on FRU continue resource_id = disk_specific_info.get("resource_id", "") if disk_specific_info.get(resource_id): assert (disk_specific_info.get(resource_id).get("ERROR") is not None) # Skip any validation on specific info if ERROR seen on sensor continue sensor_type = [ k if k.startswith("Sensor Type") else None for k in disk_specific_info.keys() ][0] assert (sensor_type is not None) assert (disk_specific_info.get("resource_id") is not None) if "States Asserted" in disk_specific_info: assert (disk_specific_info.get("States Asserted") is not None) else: # Skip any validation if ERROR seen on the specifc FRU if not disk_specific_infos.get("ERROR"): sensor_type = [ k if k.startswith("Sensor Type") else None for k in disk_specific_infos.keys() ][0] assert (sensor_type is not None) assert (disk_specific_infos.get("resource_id") is not None) if "States Asserted" in disk_specific_infos: assert (disk_specific_infos.get("States Asserted") is not None)
def test_real_stor_fan_module_actuator(agrs): check_sspl_ll_is_running() instance_id = "4" resource_type = "enclosure:fru:fan" send_enclosure_request("ENCL:%s" % resource_type, instance_id) ingressMsg = get_fru_response(resource_type, instance_id) fan_module_actuator_msg = ingressMsg.get("actuator_response_type") assert (fan_module_actuator_msg is not None) assert (fan_module_actuator_msg.get("alert_type") is not None) assert (fan_module_actuator_msg.get("alert_id") is not None) assert (fan_module_actuator_msg.get("severity") is not None) assert (fan_module_actuator_msg.get("host_id") is not None) assert (fan_module_actuator_msg.get("info") is not None) fan_module_info = fan_module_actuator_msg.get("info") assert (fan_module_info.get("site_id") is not None) assert (fan_module_info.get("node_id") is not None) assert (fan_module_info.get("cluster_id") is not None) assert (fan_module_info.get("rack_id") is not None) assert (fan_module_info.get("resource_type") is not None) assert (fan_module_info.get("event_time") is not None) assert (fan_module_info.get("resource_id") is not None) fru_specific_info = fan_module_actuator_msg.get("specific_info", {}) resource_id = fan_module_info.get("resource_id") if resource_id == "*": verify_fan_module_specific_info(fru_specific_info) return if fru_specific_info: assert (fru_specific_info.get("durable_id") is not None) assert (fru_specific_info.get("status") is not None) assert (fru_specific_info.get("name") is not None) assert (fru_specific_info.get("enclosure_id") is not None) assert (fru_specific_info.get("health") is not None) assert (fru_specific_info.get("health_reason") is not None) assert (fru_specific_info.get("location") is not None) assert (fru_specific_info.get("health_recommendation") is not None) assert (fru_specific_info.get("position") is not None) fans = fan_module_actuator_msg.get("specific_info").get("fans", []) if fans: for fan in fans: assert (fan.get("durable_id") is not None) assert (fan.get("status") is not None) assert (fan.get("name") is not None) assert (fan.get("speed") is not None) assert (fan.get("locator_led") is not None) assert (fan.get("position") is not None) assert (fan.get("location") is not None) assert (fan.get("part_number") is not None) assert (fan.get("serial_number") is not None) assert (fan.get("fw_revision") is not None) assert (fan.get("hw_revision") is not None) assert (fan.get("health") is not None) assert (fan.get("health_reason") is not None) assert (fan.get("health_recommendation") is not None)
def test_bmc_interface(args): check_sspl_ll_is_running() # backup active bmc interface BMC_IF_CONSUL_KEY, BMC_IF_CONSUL_VAL = backup_bmc_config() if BMC_IF_CONSUL_VAL == "lan": simulate_bmc_interface_alert.lan_channel_alert(BMC_IF_CONSUL_KEY, BMC_IF_CONSUL_VAL) else: simulate_bmc_interface_alert.kcs_channel_alert(BMC_IF_CONSUL_KEY, BMC_IF_CONSUL_VAL) bmc_interface_message = None for i in range(30): if world.sspl_modules[ IngressProcessorTests.name()]._is_my_msgQ_empty(): time.sleep(2) while not world.sspl_modules[ IngressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ IngressProcessorTests.name()]._read_my_msgQ() print("Received: %s" % ingressMsg) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("sensor_response_type") if msg_type["info"]["resource_type"] == "node:bmc:interface:kcs" or \ msg_type["info"]["resource_type"] == "node:bmc:interface:rmcp": bmc_interface_message = msg_type break except Exception as exception: print(exception) if bmc_interface_message: break time.sleep(1) #restore bmc config and activate ipmisimtool simulate_bmc_interface_alert.restore_config() assert (bmc_interface_message is not None) assert (bmc_interface_message.get("alert_type") is not None) alert_type = bmc_interface_message.get("alert_type") assert (alert_type == "fault") assert (bmc_interface_message.get("alert_id") is not None) assert (bmc_interface_message.get("severity") is not None) assert (bmc_interface_message.get("host_id") is not None) assert (bmc_interface_message.get("info") is not None) bmc_interface_info = bmc_interface_message.get("info") assert (bmc_interface_info.get("site_id") is not None) assert (bmc_interface_info.get("rack_id") is not None) assert (bmc_interface_info.get("node_id") is not None) assert (bmc_interface_info.get("cluster_id") is not None) assert (bmc_interface_info.get("resource_id") is not None) assert (bmc_interface_info.get("description") is not None) bmc_interface_specific_info = bmc_interface_message.get("specific_info") if bmc_interface_specific_info: assert (bmc_interface_specific_info.get("channel info") is not None)
def test_node_disk_module_actuator(agrs): print("Enters into test_node_disk_module_actuator") check_sspl_ll_is_running() time.sleep(30) instance_id = "*" disk_actuator_message_request("NDHW:node:fru:disk", instance_id) disk_actuator_msg = None ingressMsg = {} for i in range(10): if world.sspl_modules[ IngressProcessorTests.name()]._is_my_msgQ_empty(): time.sleep(2) while not world.sspl_modules[ IngressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ IngressProcessorTests.name()]._read_my_msgQ() print("Received: %s " % ingressMsg) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("actuator_response_type") if msg_type["info"]["resource_type"] == "node:fru:disk" and \ msg_type["info"]["resource_id"] == instance_id: # Break if condition is met disk_actuator_msg = msg_type break except Exception as exception: print(exception) if disk_actuator_msg: break time.sleep(1) assert (disk_actuator_msg is not None) assert (disk_actuator_msg.get("alert_type") is not None) # assert(disk_actuator_msg.get("alert_id") is not None) assert (disk_actuator_msg.get("severity") is not None) assert (disk_actuator_msg.get("host_id") is not None) assert (disk_actuator_msg.get("info") is not None) disk_actuator_info = disk_actuator_msg.get("info") assert (disk_actuator_info.get("site_id") is not None) assert (disk_actuator_info.get("node_id") is not None) # assert(disk_actuator_info.get("cluster_id") is not None) assert (disk_actuator_info.get("rack_id") is not None) assert (disk_actuator_info.get("resource_type") is not None) assert (disk_actuator_info.get("event_time") is not None) assert (disk_actuator_info.get("resource_id") is not None) disk_actuator_specific_infos = disk_actuator_msg.get("specific_info") for disk_actuator_specific_info in disk_actuator_specific_infos: assert (disk_actuator_specific_info is not None) assert (disk_actuator_specific_info.get("Sensor Type (Discrete)") is not None) assert (disk_actuator_specific_info.get("resource_id") is not None) if "States Asserted" in disk_actuator_specific_info: assert (disk_actuator_specific_info.get("States Asserted") is not None)
def test_service_restart_case(args): check_sspl_ll_is_running() check_service_is_running(service_name) DbusServiceHandler().restart(service_name) time.sleep(WAIT_TIME) sensor_response = read_ingress_queue() check_service_is_running(service_name) if sensor_response: print(sensor_response) assert (sensor_response['info']['alert_type'] != "fault")
def test_service_failed_alert(args): check_sspl_ll_is_running() check_service_is_running(service_name) simulate_service_alerts.simulate_fault_alert() time.sleep(5) sensor_response = read_ingress_queue() assert_on_mismatch(sensor_response, "fault") simulate_service_alerts.restore_service_file() time.sleep(5) sensor_response = read_ingress_queue() assert_on_mismatch(sensor_response, "fault_resolved")
def test_node_psu_module_actuator(agrs): check_sspl_ll_is_running() instance_id = "*" psu_actuator_message_request("NDHW:node:fru:psu", instance_id) psu_module_actuator_msg = None ingressMsg = {} for i in range(10): if world.sspl_modules[ IngressProcessorTests.name()]._is_my_msgQ_empty(): time.sleep(2) while not world.sspl_modules[ IngressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ IngressProcessorTests.name()]._read_my_msgQ() print("Received: %s " % ingressMsg) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("actuator_response_type") if msg_type["info"]["resource_type"] == "node:fru:psu" and \ msg_type["info"]["resource_id"] == instance_id: psu_module_actuator_msg = msg_type break except Exception as exception: print(exception) if psu_module_actuator_msg: break time.sleep(1) assert (ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID) assert (psu_module_actuator_msg is not None) assert (psu_module_actuator_msg.get("alert_type") is not None) assert (psu_module_actuator_msg.get("severity") is not None) assert (psu_module_actuator_msg.get("host_id") is not None) assert (psu_module_actuator_msg.get("info") is not None) assert (psu_module_actuator_msg.get("instance_id") is not None) psu_module_info = psu_module_actuator_msg.get("info") assert (psu_module_info.get("site_id") is not None) assert (psu_module_info.get("node_id") is not None) assert (psu_module_info.get("rack_id") is not None) assert (psu_module_info.get("resource_type") is not None) assert (psu_module_info.get("event_time") is not None) assert (psu_module_info.get("resource_id") is not None) fru_specific_infos = psu_module_actuator_msg.get("specific_info", {}) if fru_specific_infos: for fru_specific_info in fru_specific_infos: resource_id = fru_specific_info.get("resource_id") assert (fru_specific_info.get("States Asserted") is not None) assert (fru_specific_info.get("Sensor Type (Discrete)") is not None) assert (fru_specific_info.get("resource_id") is not None)
def test_cpu_data_sensor(args): check_sspl_ll_is_running() node_data_sensor_message_request("node:os:cpu_usage") cpu_data_msg = None sleep(10) while not world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ RabbitMQingressProcessorTests.name()]._read_my_msgQ() sleep(0.1) print("Received for cpu_data: {0}".format(ingressMsg)) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("sensor_response_type") if msg_type.get("info").get( "resource_type") == "node:os:cpu_usage": cpu_data_msg = msg_type break except Exception as exception: sleep(0.1) print(exception) assert (cpu_data_msg is not None) assert (cpu_data_msg.get("alert_type") is not None) assert (cpu_data_msg.get("alert_id") is not None) assert (cpu_data_msg.get("severity") is not None) assert (cpu_data_msg.get("host_id") is not None) assert (cpu_data_msg.get("info") is not None) assert (cpu_data_msg.get("specific_info") is not None) info = cpu_data_msg.get("info") assert (info.get("site_id") is not None) assert (info.get("node_id") is not None) assert (info.get("cluster_id") is not None) assert (info.get("rack_id") is not None) assert (info.get("resource_type") is not None) assert (info.get("event_time") is not None) assert (info.get("resource_id") is not None) assert (info.get("description") is not None) specific_info = cpu_data_msg.get("specific_info") assert (specific_info.get("systemTime") is not None) assert (specific_info.get("interruptTime") is not None) assert (specific_info.get("userTime") is not None) assert (specific_info.get("idleTime") is not None) assert (specific_info.get("csps") is not None) assert (specific_info.get("iowaitTime") is not None) assert (specific_info.get("niceTime") is not None) assert (specific_info.get("cpu_usage") is not None) assert (specific_info.get("coreData") is not None) assert (specific_info.get("localtime") is not None) assert (specific_info.get("softirqTime") is not None) assert (specific_info.get("stealTime") is not None)
def test_service_inactive_alert(args): check_sspl_ll_is_running() check_service_is_running(service_name) # Simulate Fault alert by stopping the service. DbusServiceHandler().stop(service_name) time.sleep(WAIT_TIME) sensor_response = read_ingress_queue() assert_on_mismatch(sensor_response, "fault") # Simulate Fault resolved alert. DbusServiceHandler().start(service_name) time.sleep(5) sensor_response = read_ingress_queue() assert_on_mismatch(sensor_response, "fault_resolved")
def test_if_data_sensor(args): check_sspl_ll_is_running() node_data_sensor_message_request(resource_type) if_data_msg = None #create dummy interface to get network alerts mock_eth_interface.shuffle_nw_interface() # Wait untill expected resource type found in RMQ ingress processor msgQ. start_time = time.time() max_wait_time = 60 while not if_data_msg: if not world.sspl_modules[ IngressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ IngressProcessorTests.name()]._read_my_msgQ() print("Received: {0}".format(ingressMsg)) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("sensor_response_type") if msg_type["info"]["resource_type"] == resource_type: if_data_msg = msg_type except Exception as exception: print(exception) if (time.time() - start_time) > max_wait_time: break assert (if_data_msg is not None) assert (if_data_msg.get("alert_type") is not None) assert (if_data_msg.get("alert_id") is not None) assert (if_data_msg.get("severity") is not None) assert (if_data_msg.get("host_id") is not None) assert (if_data_msg.get("info") is not None) assert (if_data_msg.get("specific_info") is not None) if_data_info = if_data_msg.get("info") assert (if_data_info.get("site_id") is not None) assert (if_data_info.get("node_id") is not None) assert (if_data_info.get("cluster_id") is not None) assert (if_data_info.get("rack_id") is not None) assert (if_data_info.get("resource_type") == resource_type) assert (if_data_info.get("event_time") is not None) assert (if_data_info.get("resource_id") is not None) assert (if_data_info.get("description") is not None) if_data_specific_info = if_data_msg.get("specific_info") assert (if_data_specific_info is not None) assert (if_data_specific_info.get("localtime") is not None) assert (if_data_specific_info.get("interfaces") is not None) # Remove interface added for test purpose mock_eth_interface.delete_nw_interface()
def test_iem_alerts(self): """Test iem 'ipmitool' fault alert receive.""" check_sspl_ll_is_running() Iem().iem_fault("IPMITOOL_ERROR") time.sleep(10) EventMessage.subscribe(component='sspl') fault_alert = EventMessage.receive() print(f"IEM Received:{fault_alert}") assert (fault_alert is not None) assert (fault_alert["iem"]["info"]["severity"] is not None) assert (fault_alert["iem"]["info"]["type"] is not None) assert (fault_alert["iem"]["info"]["event_time"] is not None) assert (fault_alert["iem"]["source"]["module"] is not None) assert (fault_alert["iem"]["contents"]["event"] is not None)
def test_real_stor_enclosure_sensor(agrs): timeout = time.time() + 60 * 3 check_sspl_ll_is_running() kill_mock_server() encl_sensor_message_request("enclosure") encl_sensor_msg = None while time.time() < timeout: if world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): time.sleep(1) continue ingressMsg = world.sspl_modules[ RabbitMQingressProcessorTests.name()]._read_my_msgQ() time.sleep(0.1) print("Received: %s" % ingressMsg) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("sensor_response_type") if msg_type["info"]["resource_type"] == "enclosure": encl_sensor_msg = ingressMsg.get("sensor_response_type") break except Exception as exception: time.sleep(0.1) print(exception) if encl_sensor_msg: break assert (encl_sensor_msg is not None ), "Timeout error, Real Store Enclosure Sensor test is failed." assert (encl_sensor_msg.get("alert_type") is not None) assert (encl_sensor_msg.get("alert_id") is not None) assert (encl_sensor_msg.get("severity") is not None) assert (encl_sensor_msg.get("host_id") is not None) assert (encl_sensor_msg.get("info") is not None) encl_sensor_info = encl_sensor_msg.get("info") assert (encl_sensor_info.get("site_id") is not None) assert (encl_sensor_info.get("rack_id") is not None) assert (encl_sensor_info.get("node_id") is not None) assert (encl_sensor_info.get("cluster_id") is not None) assert (encl_sensor_info.get("resource_id") is not None) assert (encl_sensor_info.get("resource_type") is not None) assert (encl_sensor_info.get("event_time") is not None) assert (encl_sensor_info.get("description") is not None) encl_specific_info = encl_sensor_msg.get("specific_info") if encl_specific_info: assert (encl_specific_info.get("event") is not None)
def test_host_update_data_sensor(args): check_sspl_ll_is_running() node_data_sensor_message_request("node:os:memory_usage") host_update_msg = None sleep(10) while not world.sspl_modules[ IngressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ IngressProcessorTests.name()]._read_my_msgQ() sleep(0.1) print("Received for host_data: {0}".format(ingressMsg)) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("sensor_response_type") if msg_type.get("info").get( "resource_type") == "node:os:memory_usage": host_update_msg = msg_type break except Exception as exception: sleep(0.1) print(exception) assert (host_update_msg is not None) assert (host_update_msg.get("alert_type") is not None) assert (host_update_msg.get("alert_id") is not None) assert (host_update_msg.get("severity") is not None) assert (host_update_msg.get("host_id") is not None) assert (host_update_msg.get("info") is not None) assert (host_update_msg.get("specific_info") is not None) info = host_update_msg.get("info") assert (info.get("site_id") is not None) assert (info.get("node_id") is not None) assert (info.get("cluster_id") is not None) assert (info.get("rack_id") is not None) assert (info.get("resource_type") is not None) assert (info.get("event_time") is not None) assert (info.get("resource_id") is not None) assert (info.get("description") is not None) specific_info = host_update_msg.get("specific_info") assert (specific_info.get("loggedInUsers") is not None) assert (specific_info.get("totalMemory") is not None) assert (specific_info.get("runningProcessCount") is not None) assert (specific_info.get("uname") is not None) assert (specific_info.get("bootTime") is not None) assert (specific_info.get("processCount") is not None) assert (specific_info.get("localtime") is not None)
def test_systemd_service_invalid_request(args): service_name = "temp_dummy.service" request = "start" check_sspl_ll_is_running() service_actuator_request(service_name, request) service_actuator_msg = None ingressMsg = {} for i in range(10): if world.sspl_modules[ IngressProcessorTests.name()]._is_my_msgQ_empty(): time.sleep(2) while not world.sspl_modules[IngressProcessorTests.name()]\ ._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[IngressProcessorTests.name()]\ ._read_my_msgQ() print("Received: %s " % ingressMsg) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("actuator_response_type") if msg_type["info"]["resource_type"] == RESOURCE_TYPE and \ msg_type["info"]["resource_id"] == service_name: # Break if required condition met service_actuator_msg = msg_type break except Exception as exception: print(exception) if service_actuator_msg: break time.sleep(1) assert (service_actuator_msg is not None) assert (service_actuator_msg.get("alert_type") == "UPDATE") assert (service_actuator_msg.get("severity") is not None) assert (service_actuator_msg.get("host_id") is not None) assert (service_actuator_msg.get("info") is not None) info = service_actuator_msg.get("info") assert (info.get("site_id") is not None) assert (info.get("node_id") is not None) assert (info.get("cluster_id") is not None) assert (info.get("rack_id") is not None) assert (info.get("resource_type") == RESOURCE_TYPE) assert (info.get("event_time") is not None) assert (info.get("resource_id") == service_name) assert (service_actuator_msg.get("specific_info") is not None) specific_info = service_actuator_msg.get("specific_info") assert (specific_info[0].get("error_msg") is not None)
def test_raid_integrity_sensor(args): platform = check_os_platform() if platform == "vm": return check_sspl_ll_is_running() node_data_sensor_message_request("node:os:raid_integrity") raid_data_msg = None sleep(10) while not world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ RabbitMQingressProcessorTests.name()]._read_my_msgQ() sleep(0.1) print("Received for raid_integrity_data: {0}".format(ingressMsg)) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("sensor_response_type") if msg_type.get("info").get( "resource_type") == "node:os:raid_integrity": raid_data_msg = msg_type break except Exception as exception: sleep(0.1) print(exception) assert (raid_data_msg is not None) assert (raid_data_msg.get("alert_type") is not None) assert (raid_data_msg.get("alert_id") is not None) assert (raid_data_msg.get("severity") is not None) assert (raid_data_msg.get("host_id") is not None) assert (raid_data_msg.get("info") is not None) assert (raid_data_msg.get("specific_info") is not None) info = raid_data_msg.get("info") assert (info.get("site_id") is not None) assert (info.get("node_id") is not None) assert (info.get("cluster_id") is not None) assert (info.get("rack_id") is not None) assert (info.get("resource_type") is not None) assert (info.get("event_time") is not None) assert (info.get("resource_id") is not None) assert (info.get("description") is not None) specific_info = raid_data_msg.get("specific_info") assert (specific_info.get("error") is None)
def test_node_voltage_sensor(agrs): check_sspl_ll_is_running() instance_id = "*" resource_type = "node:sensor:voltage" send_node_controller_message_request(UUID, "NDHW:%s" % resource_type, instance_id) ingressMsg = get_fru_response(resource_type, instance_id) assert(ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID) sensor_msg = ingressMsg.get("actuator_response_type") assert(sensor_msg is not None) assert(sensor_msg.get("alert_type") is not None) assert(sensor_msg.get("severity") is not None) assert(sensor_msg.get("host_id") is not None) assert(sensor_msg.get("info") is not None) assert(sensor_msg.get("instance_id") == instance_id) volt_module_info = sensor_msg.get("info") assert(volt_module_info.get("site_id") is not None) assert(volt_module_info.get("node_id") is not None) assert(volt_module_info.get("rack_id") is not None) assert(volt_module_info.get("resource_type") is not None) assert(volt_module_info.get("event_time") is not None) assert(volt_module_info.get("resource_id") is not None) fru_specific_infos = sensor_msg.get("specific_info") assert(fru_specific_infos is not None) for fru_specific_info in fru_specific_infos: assert(fru_specific_info is not None) if fru_specific_info.get("ERROR"): # Skip any validation on specific info if ERROR seen on FRU continue assert(fru_specific_info.get("resource_id") is not None) resource_id = fru_specific_info.get("resource_id") if fru_specific_info.get(resource_id): assert(fru_specific_info.get(resource_id).get("ERROR") is not None) # Skip any validation on specific info if ERROR seen on sensor continue assert(fru_specific_info.get("sensor_number") is not None) assert(fru_specific_info.get("sensor_status") is not None) assert(fru_specific_info.get("entity_id_instance") is not None) assert(fru_specific_info.get("sensor_reading") is not None)
def test_real_stor_dg_sensor(agrs): check_sspl_ll_is_running() dg_sensor_message_request("enclosure:cortx:disk_group") ingress_msg_type = "sensor_response_type" ingress_msg = get_fru_response("enclosure:cortx:disk_group", None, ingress_msg_type) dg_sensor_msg = ingress_msg.get(ingress_msg_type) assert (dg_sensor_msg is not None) assert (dg_sensor_msg.get("alert_type") is not None) assert (dg_sensor_msg.get("alert_id") is not None) assert (dg_sensor_msg.get("severity") is not None) assert (dg_sensor_msg.get("host_id") is not None) assert (dg_sensor_msg.get("info") is not None) dg_sensor_info = dg_sensor_msg.get("info") assert (dg_sensor_info.get("resource_type") is not None) assert (dg_sensor_info.get("event_time") is not None) assert (dg_sensor_info.get("resource_id") is not None) dg_sensor_specific_info = dg_sensor_msg.get("specific_info") assert (dg_sensor_specific_info is not None) assert (dg_sensor_specific_info.get("cvg_name") == 'A01') assert (dg_sensor_specific_info.get("cvg_id") == 0) assert (dg_sensor_specific_info.get("object_name") is not None) assert (dg_sensor_specific_info.get("name") is not None) assert (dg_sensor_specific_info.get("size") is not None) assert (dg_sensor_specific_info.get("freespace") is not None) assert (dg_sensor_specific_info.get("storage_type") is not None) assert (dg_sensor_specific_info.get("pool") is not None) assert (dg_sensor_specific_info.get("pool_serial_number") is not None) assert (dg_sensor_specific_info.get("pool_percentage") is not None) assert (dg_sensor_specific_info.get("owner") is not None) assert (dg_sensor_specific_info.get("raidtype") is not None) assert (dg_sensor_specific_info.get("status") is not None) assert (dg_sensor_specific_info.get("create_date") is not None) assert (dg_sensor_specific_info.get("disk_description") is not None) assert (dg_sensor_specific_info.get("serial_number") is not None) assert (dg_sensor_specific_info.get("pool_sector_format") is not None) assert (dg_sensor_specific_info.get("health") is not None) assert (dg_sensor_specific_info.get("health_reason") is not None) assert (dg_sensor_specific_info.get("health_recommendation") is not None)
def test_cpu_data_sensor(args): resource_type = "node:os:cpu_usage" instance_id = "*" ingress_msg_type="sensor_response_type" check_sspl_ll_is_running() node_data_sensor_message_request(resource_type) ingress_msg = get_fru_response( resource_type, instance_id, ingress_msg_type) cpu_data_msg = ingress_msg.get(ingress_msg_type) assert(cpu_data_msg is not None) assert(cpu_data_msg.get("alert_type") is not None) assert(cpu_data_msg.get("alert_id") is not None) assert(cpu_data_msg.get("severity") is not None) assert(cpu_data_msg.get("host_id") is not None) assert(cpu_data_msg.get("info") is not None) assert(cpu_data_msg.get("specific_info") is not None) info = cpu_data_msg.get("info") assert(info.get("site_id") is not None) assert(info.get("node_id") is not None) assert(info.get("cluster_id") is not None) assert(info.get("rack_id") is not None) assert(info.get("resource_type") is not None) assert(info.get("event_time") is not None) assert(info.get("resource_id") is not None) assert(info.get("description") is not None) specific_info = cpu_data_msg.get("specific_info") assert(specific_info.get("systemTime") is not None) assert(specific_info.get("interruptTime") is not None) assert(specific_info.get("userTime") is not None) assert(specific_info.get("idleTime") is not None) assert(specific_info.get("csps") is not None) assert(specific_info.get("iowaitTime") is not None) assert(specific_info.get("niceTime") is not None) assert(specific_info.get("cpu_usage") is not None) assert(specific_info.get("coreData") is not None) assert(specific_info.get("localtime") is not None) assert(specific_info.get("softirqTime") is not None) assert(specific_info.get("stealTime") is not None)
def test_systemd_service_valid_request(args): service_name = "rsyslog.service" request = "status" check_sspl_ll_is_running() # TODO: Change service name, once get final 3rd party service name service_actuator_request(service_name, request) service_actuator_msg = None time.sleep(8) ingressMsg = {} while not world.sspl_modules[IngressProcessorTests.name()]\ ._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[IngressProcessorTests.name()]\ ._read_my_msgQ() time.sleep(0.1) print("Received: %s " % ingressMsg) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("actuator_response_type") if msg_type["info"]["resource_type"] == RESOURCE_TYPE: service_actuator_msg = msg_type break except Exception as exception: time.sleep(0.1) print(exception) assert(service_actuator_msg is not None) assert(service_actuator_msg.get("alert_type") == "UPDATE") assert(service_actuator_msg.get("severity") is not None) assert(service_actuator_msg.get("host_id") is not None) assert(service_actuator_msg.get("info") is not None) info = service_actuator_msg.get("info") assert(info.get("site_id") is not None) assert(info.get("node_id") is not None) assert(info.get("cluster_id") is not None) assert(info.get("rack_id") is not None) assert(info.get("resource_type") == RESOURCE_TYPE) assert(info.get("event_time") is not None) assert(info.get("resource_id") == service_name) assert(service_actuator_msg.get("specific_info") is not None)
def test_self_hw_node_sel_event(args): check_sspl_ll_is_running() # fetch a good resource fron ipmitool result = run_cmd('ipmitool sdr type Fan') test_resource = None if result: for resource in result: if 'ok' in resource.decode().lower(): # this is the first ok resource, use it test_resource = resource.decode().split('|')[0].strip() break # inject event into sel list and wait for alert print(f"Using test resource {test_resource}") run_cmd(f"ipmitool event '{test_resource}' lcr") # wait for fault alert start_time = time.time() asserted = False while time.time() - start_time < 60: # wait for 60 seconds if wait_for_asserted_event(): asserted = True break if not asserted: print("Did not get asserted event alert.") assert (False) # revert the event run_cmd(f"ipmitool event '{test_resource}' lcr deassert") # wait for alert start_time = time.time() deasserted = False while time.time() - start_time < 60: # wait for 60 seconds if wait_for_deasserted_event(): deasserted = True break if not deasserted: print("Did not get asserted event alert.") assert (False) else: print("ipmitool returned no results.") assert (False)
def test_host_update_data_sensor(args): resource_type = "node:os:memory_usage" instance_id = "*" ingress_msg_type="sensor_response_type" check_sspl_ll_is_running() node_data_sensor_message_request(resource_type) ingress_msg = get_fru_response( resource_type, instance_id, ingress_msg_type) host_update_msg = ingress_msg.get(ingress_msg_type) assert(host_update_msg is not None) assert(host_update_msg.get("alert_type") is not None) assert(host_update_msg.get("alert_id") is not None) assert(host_update_msg.get("severity") is not None) assert(host_update_msg.get("host_id") is not None) assert(host_update_msg.get("info") is not None) assert(host_update_msg.get("specific_info") is not None) info = host_update_msg.get("info") assert(info.get("site_id") is not None) assert(info.get("node_id") is not None) assert(info.get("cluster_id") is not None) assert(info.get("rack_id") is not None) assert(info.get("resource_type") is not None) assert(info.get("event_time") is not None) assert(info.get("resource_id") is not None) assert(info.get("description") is not None) specific_info = host_update_msg.get("specific_info") assert(specific_info.get("loggedInUsers") is not None) assert(specific_info.get("totalMemory") is not None) assert(specific_info.get("runningProcessCount") is not None) assert(specific_info.get("uname") is not None) assert(specific_info.get("bootTime") is not None) assert(specific_info.get("processCount") is not None) assert(specific_info.get("localtime") is not None)
def test_real_stor_sideplane_module_actuator(agrs): check_sspl_ll_is_running() # sideplane_actuator_message_request("ENCL:enclosure:fru:sideplane", "Left Sideplane") sideplane_actuator_message_request("ENCL:enclosure:fru:sideplane", "*") sideplane_module_actuator_msg = None time.sleep(4) while not world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ RabbitMQingressProcessorTests.name()]._read_my_msgQ() time.sleep(0.1) print("Received: %s" % ingressMsg) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("actuator_response_type") if msg_type["info"]["resource_type"] == "enclosure:fru:sideplane": sideplane_module_actuator_msg = msg_type break except Exception as exception: time.sleep(0.1) print(exception) assert (sideplane_module_actuator_msg is not None) assert (sideplane_module_actuator_msg.get("alert_type") is not None) assert (sideplane_module_actuator_msg.get("alert_id") is not None) assert (sideplane_module_actuator_msg.get("severity") is not None) assert (sideplane_module_actuator_msg.get("host_id") is not None) assert (sideplane_module_actuator_msg.get("info") is not None) sideplane_module_info = sideplane_module_actuator_msg.get("info") assert (sideplane_module_info.get("site_id") is not None) assert (sideplane_module_info.get("node_id") is not None) assert (sideplane_module_info.get("cluster_id") is not None) assert (sideplane_module_info.get("rack_id") is not None) assert (sideplane_module_info.get("resource_type") is not None) assert (sideplane_module_info.get("event_time") is not None) assert (sideplane_module_info.get("resource_id") is not None) sideplane_specific_info = sideplane_module_actuator_msg.get( "specific_info", {}) resource_id = sideplane_module_info.get("resource_id") if resource_id == "*": verify_sideplane_module_specific_info(sideplane_specific_info) return if sideplane_specific_info: assert (sideplane_specific_info.get("object_name") is not None) assert (sideplane_specific_info.get("durable_id") is not None) assert (sideplane_specific_info.get("status") is not None) assert (sideplane_specific_info.get("name") is not None) assert (sideplane_specific_info.get("enclosure_id") is not None) assert (sideplane_specific_info.get("drawer_id") is not None) assert (sideplane_specific_info.get("dom_id") is not None) assert (sideplane_specific_info.get("path_id") is not None) assert (sideplane_specific_info.get("path_id_numeric") is not None) assert (sideplane_specific_info.get("location") is not None) assert (sideplane_specific_info.get("position") is not None) assert (sideplane_specific_info.get("position_numeric") is not None) assert (sideplane_specific_info.get("status_numeric") is not None) assert (sideplane_specific_info.get("extended_status") is not None) assert (sideplane_specific_info.get("health") is not None) assert (sideplane_specific_info.get("health_numeric") is not None) assert (sideplane_specific_info.get("health_reason") is not None) assert (sideplane_specific_info.get("health_recommendation") is not None) expanders = sideplane_module_actuator_msg.get("specific_info").get( "sideplanes", []) if expanders: for expander in expanders: assert (expander.get("object_name") is not None) assert (expander.get("durable_id") is not None) assert (expander.get("status") is not None) assert (expander.get("name") is not None) assert (expander.get("enclosure_id") is not None) assert (expander.get("drawer_id") is not None) assert (expander.get("dom_id") is not None) assert (expander.get("path_id") is not None) assert (expander.get("path_id_numeric") is not None) assert (expander.get("location") is not None) assert (expander.get("status_numeric") is not None) assert (expander.get("extended_status") is not None) assert (expander.get("fw_revision") is not None) assert (expander.get("health") is not None) assert (expander.get("health_numeric") is not None) assert (expander.get("health_reason") is not None) assert (expander.get("health_recommendation") is not None)
def test_node_fan_module_actuator(agrs): check_sspl_ll_is_running() instance_id = "*" resource_type = "node:fru:fan" send_node_controller_message_request(UUID, "NDHW:%s" % resource_type, instance_id) ingressMsg = get_fru_response(resource_type, instance_id) assert (ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID) fan_module_actuator_msg = ingressMsg.get("actuator_response_type") assert (fan_module_actuator_msg is not None) assert (fan_module_actuator_msg.get("alert_type") is not None) assert (fan_module_actuator_msg.get("severity") is not None) assert (fan_module_actuator_msg.get("host_id") is not None) assert (fan_module_actuator_msg.get("info") is not None) assert (fan_module_actuator_msg.get("instance_id") == instance_id) fan_module_info = fan_module_actuator_msg.get("info") assert (fan_module_info.get("site_id") is not None) assert (fan_module_info.get("node_id") is not None) assert (fan_module_info.get("rack_id") is not None) assert (fan_module_info.get("resource_type") is not None) assert (fan_module_info.get("event_time") is not None) assert (fan_module_info.get("resource_id") is not None) fan_specific_infos = fan_module_actuator_msg.get("specific_info") assert (fan_specific_infos is not None) if fan_module_info.get("resource_id") == "*": for fan_specific_info in fan_specific_infos: assert (fan_specific_info is not None) if fan_specific_info.get("ERROR"): # Skip any validation on specific info if ERROR seen on FRU continue resource_id = fan_specific_info.get("resource_id", "") if fan_specific_info.get(resource_id): assert (fan_specific_info.get(resource_id).get("ERROR") is not None) # Skip any validation on specific info if ERROR seen on sensor continue if "Fan Fail" in resource_id or "Fan Redundancy" in resource_id: sensor_type = [ k if k.startswith("Sensor Type") else None for k in fan_specific_info.keys() ][0] assert (sensor_type is not None) assert (fan_specific_info.get("resource_id") is not None) assert (fan_specific_info.get("States Asserted") is not None) else: assert (fan_specific_info.get("resource_id") is not None) sensor_type = [ k if k.startswith("Sensor Type") else None for k in fan_specific_info.keys() ][0] assert (sensor_type is not None) assert (fan_specific_info.get("Sensor Reading") is not None) assert (fan_specific_info.get("Status") is not None) assert (fan_specific_info.get("Lower Non_Recoverable") is not None) assert (fan_specific_info.get("Lower Critical") is not None) assert (fan_specific_info.get("Lower Non_Critical") is not None) assert (fan_specific_info.get("Upper Non_Critical") is not None) assert (fan_specific_info.get("Upper Critical") is not None) assert (fan_specific_info.get("Upper Non_Recoverable") is not None) assert (fan_specific_info.get("Positive Hysteresis") is not None) assert (fan_specific_info.get("Negative Hysteresis") is not None) assert (fan_specific_info.get("Assertion Events") is not None) assert (fan_specific_info.get("Assertions Enabled") is not None) assert (fan_specific_info.get("Deassertions Enabled") is not None) else: # Skip any validation if ERROR seen on the specifc FRU if not fan_specific_infos.get("ERROR"): sensor_type = [ k if k.startswith("Sensor Type") else None for k in fan_specific_infos.keys() ][0] assert (sensor_type is not None) assert (fan_specific_infos.get("Sensor Reading") is not None) assert (fan_specific_infos.get("Status") is not None) assert (fan_specific_infos.get("Lower Non_Recoverable") is not None) assert (fan_specific_infos.get("Lower Critical") is not None) assert (fan_specific_infos.get("Lower Non_Critical") is not None) assert (fan_specific_infos.get("Upper Non_Critical") is not None) assert (fan_specific_infos.get("Upper Critical") is not None) assert (fan_specific_infos.get("Upper Non_Recoverable") is not None) assert (fan_specific_infos.get("Positive Hysteresis") is not None) assert (fan_specific_infos.get("Negative Hysteresis") is not None) assert (fan_specific_infos.get("Assertion Events") is not None) assert (fan_specific_infos.get("Assertions Enabled") is not None) assert (fan_specific_infos.get("Deassertions Enabled") is not None) assert (fan_specific_infos.get("resource_id") is not None)
def test_real_stor_sideplane_module_actuator(agrs): check_sspl_ll_is_running() instance_id = "*" resource_type = "enclosure:hw:sideplane" send_enclosure_request("ENCL:%s" % resource_type, instance_id) ingressMsg = get_fru_response(resource_type, instance_id) sideplane_module_actuator_msg = ingressMsg.get("actuator_response_type") assert (sideplane_module_actuator_msg is not None) assert (sideplane_module_actuator_msg.get("alert_type") is not None) assert (sideplane_module_actuator_msg.get("alert_id") is not None) assert (sideplane_module_actuator_msg.get("severity") is not None) assert (sideplane_module_actuator_msg.get("host_id") is not None) assert (sideplane_module_actuator_msg.get("info") is not None) sideplane_module_info = sideplane_module_actuator_msg.get("info") assert (sideplane_module_info.get("site_id") is not None) assert (sideplane_module_info.get("node_id") is not None) assert (sideplane_module_info.get("cluster_id") is not None) assert (sideplane_module_info.get("rack_id") is not None) assert (sideplane_module_info.get("resource_type") is not None) assert (sideplane_module_info.get("event_time") is not None) assert (sideplane_module_info.get("resource_id") is not None) sideplane_specific_info = sideplane_module_actuator_msg.get( "specific_info", {}) resource_id = sideplane_module_info.get("resource_id") if resource_id == "*": verify_sideplane_module_specific_info(sideplane_specific_info) return if sideplane_specific_info: assert (sideplane_specific_info.get("object_name") is not None) assert (sideplane_specific_info.get("durable_id") is not None) assert (sideplane_specific_info.get("status") is not None) assert (sideplane_specific_info.get("name") is not None) assert (sideplane_specific_info.get("enclosure_id") is not None) assert (sideplane_specific_info.get("drawer_id") is not None) assert (sideplane_specific_info.get("dom_id") is not None) assert (sideplane_specific_info.get("path_id") is not None) assert (sideplane_specific_info.get("path_id_numeric") is not None) assert (sideplane_specific_info.get("location") is not None) assert (sideplane_specific_info.get("position") is not None) assert (sideplane_specific_info.get("position_numeric") is not None) assert (sideplane_specific_info.get("status_numeric") is not None) assert (sideplane_specific_info.get("extended_status") is not None) assert (sideplane_specific_info.get("health") is not None) assert (sideplane_specific_info.get("health_numeric") is not None) assert (sideplane_specific_info.get("health_reason") is not None) assert (sideplane_specific_info.get("health_recommendation") is not None) expanders = sideplane_module_actuator_msg.get("specific_info").get( "sideplanes", []) if expanders: for expander in expanders: assert (expander.get("object_name") is not None) assert (expander.get("durable_id") is not None) assert (expander.get("status") is not None) assert (expander.get("name") is not None) assert (expander.get("enclosure_id") is not None) assert (expander.get("drawer_id") is not None) assert (expander.get("dom_id") is not None) assert (expander.get("path_id") is not None) assert (expander.get("path_id_numeric") is not None) assert (expander.get("location") is not None) assert (expander.get("status_numeric") is not None) assert (expander.get("extended_status") is not None) assert (expander.get("fw_revision") is not None) assert (expander.get("health") is not None) assert (expander.get("health_numeric") is not None) assert (expander.get("health_reason") is not None) assert (expander.get("health_recommendation") is not None)
def test_node_fan_module_actuator(agrs): check_sspl_ll_is_running() fan_actuator_message_request("NDHW:node:fru:fan", str(test_resource)) fan_module_actuator_msg = None ingressMsg = {} for i in range(10): if world.sspl_modules[IngressProcessorTests.name()]._is_my_msgQ_empty(): time.sleep(2) while not world.sspl_modules[IngressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[IngressProcessorTests.name()]._read_my_msgQ() time.sleep(0.1) print("Received: %s" % ingressMsg) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("actuator_response_type") if msg_type["info"]["resource_type"] == "node:fru:fan" and \ msg_type["info"]["resource_id"] == test_resource: # Break if condition is met fan_module_actuator_msg = msg_type break except Exception as exception: time.sleep(0.1) print(exception) if fan_module_actuator_msg: break time.sleep(1) assert(ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID) assert(fan_module_actuator_msg is not None) assert(fan_module_actuator_msg.get("alert_type") is not None) assert(fan_module_actuator_msg.get("severity") is not None) assert(fan_module_actuator_msg.get("host_id") is not None) assert(fan_module_actuator_msg.get("info") is not None) assert(fan_module_actuator_msg.get("instance_id") is not None) fan_module_info = fan_module_actuator_msg.get("info") assert(fan_module_info.get("site_id") is not None) assert(fan_module_info.get("node_id") is not None) assert(fan_module_info.get("rack_id") is not None) assert(fan_module_info.get("resource_type") is not None) assert(fan_module_info.get("event_time") is not None) assert(fan_module_info.get("resource_id") is not None) fru_specific_infos = fan_module_actuator_msg.get("specific_info", {}) if fru_specific_infos and fan_module_info.get("resource_id") == "*": for fru_specific_info in fru_specific_infos: resource_id = fru_specific_info.get("resource_id") if "Fan Fail" in resource_id: assert(fru_specific_info.get("Sensor Type (Discrete)") is not None) assert(fru_specific_info.get("resource_id") is not None) elif "System Fan" in resource_id: assert(fru_specific_info.get("Status") is not None) assert(fru_specific_info.get("Sensor Type (Threshold)") is not None) assert(fru_specific_info.get("Sensor Reading") is not None) assert(fru_specific_info.get("Lower Non-Recoverable") is not None) assert(fru_specific_info.get("Assertions Enabled") is not None) assert(fru_specific_info.get("Upper Non-Critical") is not None) assert(fru_specific_info.get("Upper Non-Recoverable") is not None) assert(fru_specific_info.get("Positive Hysteresis") is not None) assert(fru_specific_info.get("Lower Critical") is not None) assert(fru_specific_info.get("Deassertions Enabled") is not None) assert(fru_specific_info.get("Lower Non-Critical") is not None) assert(fru_specific_info.get("Upper Critical") is not None) assert(fru_specific_info.get("Negative Hysteresis") is not None) assert(fru_specific_info.get("Assertion Events") is not None) assert(fru_specific_info.get("resource_id") is not None) else: assert(fru_specific_info.get("States Asserted") is not None) assert(fru_specific_info.get("Sensor Type (Discrete)") is not None) assert(fru_specific_info.get("resource_id") is not None) elif fru_specific_infos: assert(fru_specific_infos.get("Sensor Type (Threshold)") is not None) assert(fru_specific_infos.get("Sensor Reading") is not None) assert(fru_specific_infos.get("Status") is not None) assert(fru_specific_infos.get("Lower Non_Recoverable") is not None) assert(fru_specific_infos.get("Lower Critical") is not None) assert(fru_specific_infos.get("Lower Non_Critical") is not None) assert(fru_specific_infos.get("Upper Non_Critical") is not None) assert(fru_specific_infos.get("Upper Critical") is not None) assert(fru_specific_infos.get("Upper Non_Recoverable") is not None) assert(fru_specific_infos.get("Positive Hysteresis") is not None) assert(fru_specific_infos.get("Negative Hysteresis") is not None) assert(fru_specific_infos.get("Assertion Events") is not None) assert(fru_specific_infos.get("Assertions Enabled") is not None) assert(fru_specific_infos.get("Deassertions Enabled") is not None) assert(fru_specific_infos.get("resource_id") is not None)