def test_node_disk_module_actuator(agrs): print("Enters into test_node_disk_module_actuator") check_sspl_ll_is_running() disk_actuator_message_request("NDHW:node:fru:disk") disk_actuator_msg = None time.sleep(10) ingressMsg = {} for i in range(10): if world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): time.sleep(2) while not world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ RabbitMQingressProcessorTests.name()]._read_my_msgQ() time.sleep(0.1) print("Received: %s " % ingressMsg) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("actuator_response_type") if msg_type["info"]["resource_type"] == "node:fru:disk": disk_actuator_msg = msg_type break except Exception as exception: time.sleep(0.1) print(exception) if disk_actuator_msg: break time.sleep(1) assert (disk_actuator_msg is not None) assert (disk_actuator_msg.get("alert_type") is not None) # assert(disk_actuator_msg.get("alert_id") is not None) assert (disk_actuator_msg.get("severity") is not None) assert (disk_actuator_msg.get("host_id") is not None) assert (disk_actuator_msg.get("info") is not None) disk_actuator_info = disk_actuator_msg.get("info") assert (disk_actuator_info.get("site_id") is not None) assert (disk_actuator_info.get("node_id") is not None) # assert(disk_actuator_info.get("cluster_id") is not None) assert (disk_actuator_info.get("rack_id") is not None) assert (disk_actuator_info.get("resource_type") is not None) assert (disk_actuator_info.get("event_time") is not None) assert (disk_actuator_info.get("resource_id") is not None) disk_actuator_specific_infos = disk_actuator_msg.get("specific_info") for disk_actuator_specific_info in disk_actuator_specific_infos: assert (disk_actuator_specific_info is not None) assert (disk_actuator_specific_info.get("Sensor Type (Discrete)") is not None) assert (disk_actuator_specific_info.get("resource_id") is not None) if "States Asserted" in disk_actuator_specific_info: assert (disk_actuator_specific_info.get("States Asserted") is not None)
def test_bmc_interface(args): check_sspl_ll_is_running() # backup active bmc interface BMC_IF_CONSUL_KEY, BMC_IF_CONSUL_VAL = backup_bmc_config() if BMC_IF_CONSUL_VAL == "lan": simulate_bmc_interface_alert.lan_channel_alert(BMC_IF_CONSUL_KEY, BMC_IF_CONSUL_VAL) else: simulate_bmc_interface_alert.kcs_channel_alert(BMC_IF_CONSUL_KEY, BMC_IF_CONSUL_VAL) bmc_interface_message = None time.sleep(25) while not world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ RabbitMQingressProcessorTests.name()]._read_my_msgQ() time.sleep(0.1) print("Received: %s" % ingressMsg) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("sensor_response_type") if msg_type["info"]["resource_type"] == "node:bmc:interface:kcs" or \ msg_type["info"]["resource_type"] == "node:bmc:interface:rmcp": bmc_interface_message = msg_type break except Exception as exception: time.sleep(0.1) print(exception) #restore bmc config and activate ipmisimtool simulate_bmc_interface_alert.restore_config() assert (bmc_interface_message is not None) assert (bmc_interface_message.get("alert_type") is not None) alert_type = bmc_interface_message.get("alert_type") assert (alert_type == "fault") assert (bmc_interface_message.get("alert_id") is not None) assert (bmc_interface_message.get("severity") is not None) assert (bmc_interface_message.get("host_id") is not None) assert (bmc_interface_message.get("info") is not None) bmc_interface_info = bmc_interface_message.get("info") assert (bmc_interface_info.get("site_id") is not None) assert (bmc_interface_info.get("rack_id") is not None) assert (bmc_interface_info.get("node_id") is not None) assert (bmc_interface_info.get("cluster_id") is not None) assert (bmc_interface_info.get("resource_id") is not None) assert (bmc_interface_info.get("description") is not None) bmc_interface_specific_info = bmc_interface_message.get("specific_info") if bmc_interface_specific_info: assert (bmc_interface_specific_info.get("channel info") is not None)
def test_cpu_data_sensor(args): check_sspl_ll_is_running() node_data_sensor_message_request("node:os:cpu_usage") cpu_data_msg = None sleep(10) while not world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ RabbitMQingressProcessorTests.name()]._read_my_msgQ() sleep(0.1) print("Received for cpu_data: {0}".format(ingressMsg)) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("sensor_response_type") if msg_type.get("info").get( "resource_type") == "node:os:cpu_usage": cpu_data_msg = msg_type break except Exception as exception: sleep(0.1) print(exception) assert (cpu_data_msg is not None) assert (cpu_data_msg.get("alert_type") is not None) assert (cpu_data_msg.get("alert_id") is not None) assert (cpu_data_msg.get("severity") is not None) assert (cpu_data_msg.get("host_id") is not None) assert (cpu_data_msg.get("info") is not None) assert (cpu_data_msg.get("specific_info") is not None) info = cpu_data_msg.get("info") assert (info.get("site_id") is not None) assert (info.get("node_id") is not None) assert (info.get("cluster_id") is not None) assert (info.get("rack_id") is not None) assert (info.get("resource_type") is not None) assert (info.get("event_time") is not None) assert (info.get("resource_id") is not None) assert (info.get("description") is not None) specific_info = cpu_data_msg.get("specific_info") assert (specific_info.get("systemTime") is not None) assert (specific_info.get("interruptTime") is not None) assert (specific_info.get("userTime") is not None) assert (specific_info.get("idleTime") is not None) assert (specific_info.get("csps") is not None) assert (specific_info.get("iowaitTime") is not None) assert (specific_info.get("niceTime") is not None) assert (specific_info.get("cpu_usage") is not None) assert (specific_info.get("coreData") is not None) assert (specific_info.get("localtime") is not None) assert (specific_info.get("softirqTime") is not None) assert (specific_info.get("stealTime") is not None)
def test_real_stor_enclosure_sensor(agrs): timeout = time.time() + 60 * 3 check_sspl_ll_is_running() kill_mock_server() encl_sensor_message_request("enclosure") encl_sensor_msg = None while time.time() < timeout: if world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): time.sleep(1) continue ingressMsg = world.sspl_modules[ RabbitMQingressProcessorTests.name()]._read_my_msgQ() time.sleep(0.1) print("Received: %s" % ingressMsg) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("sensor_response_type") if msg_type["info"]["resource_type"] == "enclosure": encl_sensor_msg = ingressMsg.get("sensor_response_type") break except Exception as exception: time.sleep(0.1) print(exception) if encl_sensor_msg: break assert (encl_sensor_msg is not None ), "Timeout error, Real Store Enclosure Sensor test is failed." assert (encl_sensor_msg.get("alert_type") is not None) assert (encl_sensor_msg.get("alert_id") is not None) assert (encl_sensor_msg.get("severity") is not None) assert (encl_sensor_msg.get("host_id") is not None) assert (encl_sensor_msg.get("info") is not None) encl_sensor_info = encl_sensor_msg.get("info") assert (encl_sensor_info.get("site_id") is not None) assert (encl_sensor_info.get("rack_id") is not None) assert (encl_sensor_info.get("node_id") is not None) assert (encl_sensor_info.get("cluster_id") is not None) assert (encl_sensor_info.get("resource_id") is not None) assert (encl_sensor_info.get("resource_type") is not None) assert (encl_sensor_info.get("event_time") is not None) assert (encl_sensor_info.get("description") is not None) encl_specific_info = encl_sensor_msg.get("specific_info") if encl_specific_info: assert (encl_specific_info.get("event") is not None)
def test_host_update_data_sensor(args): check_sspl_ll_is_running() node_data_sensor_message_request("node:os:memory_usage") host_update_msg = None sleep(10) while not world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ RabbitMQingressProcessorTests.name()]._read_my_msgQ() sleep(0.1) print("Received for host_data: {0}".format(ingressMsg)) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("sensor_response_type") if msg_type.get("info").get( "resource_type") == "node:os:memory_usage": host_update_msg = msg_type break except Exception as exception: sleep(0.1) print(exception) assert (host_update_msg is not None) assert (host_update_msg.get("alert_type") is not None) assert (host_update_msg.get("alert_id") is not None) assert (host_update_msg.get("severity") is not None) assert (host_update_msg.get("host_id") is not None) assert (host_update_msg.get("info") is not None) assert (host_update_msg.get("specific_info") is not None) info = host_update_msg.get("info") assert (info.get("site_id") is not None) assert (info.get("node_id") is not None) assert (info.get("cluster_id") is not None) assert (info.get("rack_id") is not None) assert (info.get("resource_type") is not None) assert (info.get("event_time") is not None) assert (info.get("resource_id") is not None) assert (info.get("description") is not None) specific_info = host_update_msg.get("specific_info") assert (specific_info.get("loggedInUsers") is not None) assert (specific_info.get("totalMemory") is not None) assert (specific_info.get("runningProcessCount") is not None) assert (specific_info.get("uname") is not None) assert (specific_info.get("bootTime") is not None) assert (specific_info.get("processCount") is not None) assert (specific_info.get("localtime") is not None)
def test_node_psu_module_actuator(agrs): check_sspl_ll_is_running() psu_actuator_message_request("NDHW:node:fru:psu", "*") psu_module_actuator_msg = None time.sleep(6) ingressMsg = {} while not world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ RabbitMQingressProcessorTests.name()]._read_my_msgQ() time.sleep(0.1) print("Received: %s " % ingressMsg) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("actuator_response_type") if msg_type["info"]["resource_type"] == "node:fru:psu": psu_module_actuator_msg = msg_type break except Exception as exception: time.sleep(0.1) print(exception) assert (ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID) assert (psu_module_actuator_msg is not None) assert (psu_module_actuator_msg.get("alert_type") is not None) assert (psu_module_actuator_msg.get("severity") is not None) assert (psu_module_actuator_msg.get("host_id") is not None) assert (psu_module_actuator_msg.get("info") is not None) assert (psu_module_actuator_msg.get("instance_id") is not None) psu_module_info = psu_module_actuator_msg.get("info") assert (psu_module_info.get("site_id") is not None) assert (psu_module_info.get("node_id") is not None) assert (psu_module_info.get("rack_id") is not None) assert (psu_module_info.get("resource_type") is not None) assert (psu_module_info.get("event_time") is not None) assert (psu_module_info.get("resource_id") is not None) fru_specific_infos = psu_module_actuator_msg.get("specific_info", {}) if fru_specific_infos: for fru_specific_info in fru_specific_infos: resource_id = fru_specific_info.get("resource_id") assert (fru_specific_info.get("States Asserted") is not None) assert (fru_specific_info.get("Sensor Type (Discrete)") is not None) assert (fru_specific_info.get("resource_id") is not None)
def test_if_data_sensor(args): check_sspl_ll_is_running() node_data_sensor_message_request("node:interface:nw") if_data_msg = None #create dummy interface to get network alerts mock_eth_interface.shuffle_nw_interface() sleep(10) while not world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ RabbitMQingressProcessorTests.name()]._read_my_msgQ() sleep(0.1) print("Received for if_data: {0}".format(ingressMsg)) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("sensor_response_type") if msg_type["info"]["resource_type"] == "node:interface:nw": if_data_msg = msg_type break except Exception as exception: sleep(0.1) print(exception) assert (if_data_msg is not None) assert (if_data_msg.get("alert_type") is not None) assert (if_data_msg.get("alert_id") is not None) assert (if_data_msg.get("severity") is not None) assert (if_data_msg.get("host_id") is not None) assert (if_data_msg.get("info") is not None) assert (if_data_msg.get("specific_info") is not None) if_data_info = if_data_msg.get("info") assert (if_data_info.get("site_id") is not None) assert (if_data_info.get("node_id") is not None) assert (if_data_info.get("cluster_id") is not None) assert (if_data_info.get("rack_id") is not None) assert (if_data_info.get("resource_type") is not None) assert (if_data_info.get("event_time") is not None) assert (if_data_info.get("resource_id") is not None) assert (if_data_info.get("description") is not None) if_data_specific_info = if_data_msg.get("specific_info") assert (if_data_specific_info is not None) assert (if_data_specific_info.get("localtime") is not None) assert (if_data_specific_info.get("interfaces") is not None)
def test_raid_integrity_sensor(args): platform = check_os_platform() if platform == "vm": return check_sspl_ll_is_running() node_data_sensor_message_request("node:os:raid_integrity") raid_data_msg = None sleep(10) while not world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[ RabbitMQingressProcessorTests.name()]._read_my_msgQ() sleep(0.1) print("Received for raid_integrity_data: {0}".format(ingressMsg)) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("sensor_response_type") if msg_type.get("info").get( "resource_type") == "node:os:raid_integrity": raid_data_msg = msg_type break except Exception as exception: sleep(0.1) print(exception) assert (raid_data_msg is not None) assert (raid_data_msg.get("alert_type") is not None) assert (raid_data_msg.get("alert_id") is not None) assert (raid_data_msg.get("severity") is not None) assert (raid_data_msg.get("host_id") is not None) assert (raid_data_msg.get("info") is not None) assert (raid_data_msg.get("specific_info") is not None) info = raid_data_msg.get("info") assert (info.get("site_id") is not None) assert (info.get("node_id") is not None) assert (info.get("cluster_id") is not None) assert (info.get("rack_id") is not None) assert (info.get("resource_type") is not None) assert (info.get("event_time") is not None) assert (info.get("resource_id") is not None) assert (info.get("description") is not None) specific_info = raid_data_msg.get("specific_info") assert (specific_info.get("error") is None)
def test_self_hw_node_sel_event(args): check_sspl_ll_is_running() # fetch a good resource fron ipmitool result = run_cmd('ipmitool sdr type Fan') test_resource = None if result: for resource in result: if 'ok' in resource.decode().lower(): # this is the first ok resource, use it test_resource = resource.decode().split('|')[0].strip() break # inject event into sel list and wait for alert print(f"Using test resource {test_resource}") run_cmd(f"ipmitool event '{test_resource}' lcr") # wait for fault alert start_time = time.time() asserted = False while time.time() - start_time < 60: # wait for 60 seconds if wait_for_asserted_event(): asserted = True break if not asserted: print("Did not get asserted event alert.") assert (False) # revert the event run_cmd(f"ipmitool event '{test_resource}' lcr deassert") # wait for alert start_time = time.time() deasserted = False while time.time() - start_time < 60: # wait for 60 seconds if wait_for_deasserted_event(): deasserted = True break if not deasserted: print("Did not get asserted event alert.") assert (False) else: print("ipmitool returned no results.") assert (False)
def test_node_fan_module_actuator(agrs): check_sspl_ll_is_running() fan_actuator_message_request("NDHW:node:fru:fan", str(test_resource)) fan_module_actuator_msg = None time.sleep(6) ingressMsg = {} while not world.sspl_modules[RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[RabbitMQingressProcessorTests.name()]._read_my_msgQ() time.sleep(0.1) print("Received: %s" % ingressMsg) try: # Make sure we get back the message type that matches the request msg_type = ingressMsg.get("actuator_response_type") if msg_type["info"]["resource_type"] == "node:fru:fan": fan_module_actuator_msg = msg_type break except Exception as exception: time.sleep(0.1) print(exception) assert(ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID) assert(fan_module_actuator_msg is not None) assert(fan_module_actuator_msg.get("alert_type") is not None) assert(fan_module_actuator_msg.get("severity") is not None) assert(fan_module_actuator_msg.get("host_id") is not None) assert(fan_module_actuator_msg.get("info") is not None) assert(fan_module_actuator_msg.get("instance_id") is not None) fan_module_info = fan_module_actuator_msg.get("info") assert(fan_module_info.get("site_id") is not None) assert(fan_module_info.get("node_id") is not None) assert(fan_module_info.get("rack_id") is not None) assert(fan_module_info.get("resource_type") is not None) assert(fan_module_info.get("event_time") is not None) assert(fan_module_info.get("resource_id") is not None) fru_specific_infos = fan_module_actuator_msg.get("specific_info", {}) if fru_specific_infos and fan_module_info.get("resource_id") == "*": for fru_specific_info in fru_specific_infos: resource_id = fru_specific_info.get("resource_id") if "Fan Fail" in resource_id: assert(fru_specific_info.get("Sensor Type (Discrete)") is not None) assert(fru_specific_info.get("resource_id") is not None) elif "System Fan" in resource_id: assert(fru_specific_info.get("Status") is not None) assert(fru_specific_info.get("Sensor Type (Threshold)") is not None) assert(fru_specific_info.get("Sensor Reading") is not None) assert(fru_specific_info.get("Lower Non-Recoverable") is not None) assert(fru_specific_info.get("Assertions Enabled") is not None) assert(fru_specific_info.get("Upper Non-Critical") is not None) assert(fru_specific_info.get("Upper Non-Recoverable") is not None) assert(fru_specific_info.get("Positive Hysteresis") is not None) assert(fru_specific_info.get("Lower Critical") is not None) assert(fru_specific_info.get("Deassertions Enabled") is not None) assert(fru_specific_info.get("Lower Non-Critical") is not None) assert(fru_specific_info.get("Upper Critical") is not None) assert(fru_specific_info.get("Negative Hysteresis") is not None) assert(fru_specific_info.get("Assertion Events") is not None) assert(fru_specific_info.get("resource_id") is not None) else: assert(fru_specific_info.get("States Asserted") is not None) assert(fru_specific_info.get("Sensor Type (Discrete)") is not None) assert(fru_specific_info.get("resource_id") is not None) elif fru_specific_infos: assert(fru_specific_infos.get("Sensor Type (Threshold)") is not None) assert(fru_specific_infos.get("Sensor Reading") is not None) assert(fru_specific_infos.get("Status") is not None) assert(fru_specific_infos.get("Lower Non_Recoverable") is not None) assert(fru_specific_infos.get("Lower Critical") is not None) assert(fru_specific_infos.get("Lower Non_Critical") is not None) assert(fru_specific_infos.get("Upper Non_Critical") is not None) assert(fru_specific_infos.get("Upper Critical") is not None) assert(fru_specific_infos.get("Upper Non_Recoverable") is not None) assert(fru_specific_infos.get("Positive Hysteresis") is not None) assert(fru_specific_infos.get("Negative Hysteresis") is not None) assert(fru_specific_infos.get("Assertion Events") is not None) assert(fru_specific_infos.get("Assertions Enabled") is not None) assert(fru_specific_infos.get("Deassertions Enabled") is not None) assert(fru_specific_infos.get("resource_id") is not None)