def test_node_disk_module_actuator(agrs):
print("Enters into test_node_disk_module_actuator")
check_sspl_ll_is_running()
disk_actuator_message_request("NDHW:node:fru:disk")
disk_actuator_msg = None
time.sleep(10)
ingressMsg = {}
for i in range(10):
if world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
time.sleep(2)
while not world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
RabbitMQingressProcessorTests.name()]._read_my_msgQ()
time.sleep(0.1)
print("Received: %s " % ingressMsg)
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("actuator_response_type")
if msg_type["info"]["resource_type"] == "node:fru:disk":
disk_actuator_msg = msg_type
break
except Exception as exception:
time.sleep(0.1)
print(exception)
if disk_actuator_msg:
break
time.sleep(1)
assert (disk_actuator_msg is not None)
assert (disk_actuator_msg.get("alert_type") is not None)
# assert(disk_actuator_msg.get("alert_id") is not None)
assert (disk_actuator_msg.get("severity") is not None)
assert (disk_actuator_msg.get("host_id") is not None)
assert (disk_actuator_msg.get("info") is not None)
disk_actuator_info = disk_actuator_msg.get("info")
assert (disk_actuator_info.get("site_id") is not None)
assert (disk_actuator_info.get("node_id") is not None)
# assert(disk_actuator_info.get("cluster_id") is not None)
assert (disk_actuator_info.get("rack_id") is not None)
assert (disk_actuator_info.get("resource_type") is not None)
assert (disk_actuator_info.get("event_time") is not None)
assert (disk_actuator_info.get("resource_id") is not None)
disk_actuator_specific_infos = disk_actuator_msg.get("specific_info")
for disk_actuator_specific_info in disk_actuator_specific_infos:
assert (disk_actuator_specific_info is not None)
assert (disk_actuator_specific_info.get("Sensor Type (Discrete)")
is not None)
assert (disk_actuator_specific_info.get("resource_id") is not None)
if "States Asserted" in disk_actuator_specific_info:
assert (disk_actuator_specific_info.get("States Asserted")
is not None)
def test_bmc_interface(args):
check_sspl_ll_is_running()
# backup active bmc interface
BMC_IF_CONSUL_KEY, BMC_IF_CONSUL_VAL = backup_bmc_config()
if BMC_IF_CONSUL_VAL == "lan":
simulate_bmc_interface_alert.lan_channel_alert(BMC_IF_CONSUL_KEY,
BMC_IF_CONSUL_VAL)
else:
simulate_bmc_interface_alert.kcs_channel_alert(BMC_IF_CONSUL_KEY,
BMC_IF_CONSUL_VAL)
bmc_interface_message = None
time.sleep(25)
while not world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
RabbitMQingressProcessorTests.name()]._read_my_msgQ()
time.sleep(0.1)
print("Received: %s" % ingressMsg)
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("sensor_response_type")
if msg_type["info"]["resource_type"] == "node:bmc:interface:kcs" or \
msg_type["info"]["resource_type"] == "node:bmc:interface:rmcp":
bmc_interface_message = msg_type
break
except Exception as exception:
time.sleep(0.1)
print(exception)
#restore bmc config and activate ipmisimtool
simulate_bmc_interface_alert.restore_config()
assert (bmc_interface_message is not None)
assert (bmc_interface_message.get("alert_type") is not None)
alert_type = bmc_interface_message.get("alert_type")
assert (alert_type == "fault")
assert (bmc_interface_message.get("alert_id") is not None)
assert (bmc_interface_message.get("severity") is not None)
assert (bmc_interface_message.get("host_id") is not None)
assert (bmc_interface_message.get("info") is not None)
bmc_interface_info = bmc_interface_message.get("info")
assert (bmc_interface_info.get("site_id") is not None)
assert (bmc_interface_info.get("rack_id") is not None)
assert (bmc_interface_info.get("node_id") is not None)
assert (bmc_interface_info.get("cluster_id") is not None)
assert (bmc_interface_info.get("resource_id") is not None)
assert (bmc_interface_info.get("description") is not None)
bmc_interface_specific_info = bmc_interface_message.get("specific_info")
if bmc_interface_specific_info:
assert (bmc_interface_specific_info.get("channel info") is not None)
def test_cpu_data_sensor(args):
check_sspl_ll_is_running()
node_data_sensor_message_request("node:os:cpu_usage")
cpu_data_msg = None
sleep(10)
while not world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
RabbitMQingressProcessorTests.name()]._read_my_msgQ()
sleep(0.1)
print("Received for cpu_data: {0}".format(ingressMsg))
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("sensor_response_type")
if msg_type.get("info").get(
"resource_type") == "node:os:cpu_usage":
cpu_data_msg = msg_type
break
except Exception as exception:
sleep(0.1)
print(exception)
assert (cpu_data_msg is not None)
assert (cpu_data_msg.get("alert_type") is not None)
assert (cpu_data_msg.get("alert_id") is not None)
assert (cpu_data_msg.get("severity") is not None)
assert (cpu_data_msg.get("host_id") is not None)
assert (cpu_data_msg.get("info") is not None)
assert (cpu_data_msg.get("specific_info") is not None)
info = cpu_data_msg.get("info")
assert (info.get("site_id") is not None)
assert (info.get("node_id") is not None)
assert (info.get("cluster_id") is not None)
assert (info.get("rack_id") is not None)
assert (info.get("resource_type") is not None)
assert (info.get("event_time") is not None)
assert (info.get("resource_id") is not None)
assert (info.get("description") is not None)
specific_info = cpu_data_msg.get("specific_info")
assert (specific_info.get("systemTime") is not None)
assert (specific_info.get("interruptTime") is not None)
assert (specific_info.get("userTime") is not None)
assert (specific_info.get("idleTime") is not None)
assert (specific_info.get("csps") is not None)
assert (specific_info.get("iowaitTime") is not None)
assert (specific_info.get("niceTime") is not None)
assert (specific_info.get("cpu_usage") is not None)
assert (specific_info.get("coreData") is not None)
assert (specific_info.get("localtime") is not None)
assert (specific_info.get("softirqTime") is not None)
assert (specific_info.get("stealTime") is not None)
def test_real_stor_enclosure_sensor(agrs):
timeout = time.time() + 60 * 3
check_sspl_ll_is_running()
kill_mock_server()
encl_sensor_message_request("enclosure")
encl_sensor_msg = None
while time.time() < timeout:
if world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
time.sleep(1)
continue
ingressMsg = world.sspl_modules[
RabbitMQingressProcessorTests.name()]._read_my_msgQ()
time.sleep(0.1)
print("Received: %s" % ingressMsg)
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("sensor_response_type")
if msg_type["info"]["resource_type"] == "enclosure":
encl_sensor_msg = ingressMsg.get("sensor_response_type")
break
except Exception as exception:
time.sleep(0.1)
print(exception)
if encl_sensor_msg:
break
assert (encl_sensor_msg is not None
), "Timeout error, Real Store Enclosure Sensor test is failed."
assert (encl_sensor_msg.get("alert_type") is not None)
assert (encl_sensor_msg.get("alert_id") is not None)
assert (encl_sensor_msg.get("severity") is not None)
assert (encl_sensor_msg.get("host_id") is not None)
assert (encl_sensor_msg.get("info") is not None)
encl_sensor_info = encl_sensor_msg.get("info")
assert (encl_sensor_info.get("site_id") is not None)
assert (encl_sensor_info.get("rack_id") is not None)
assert (encl_sensor_info.get("node_id") is not None)
assert (encl_sensor_info.get("cluster_id") is not None)
assert (encl_sensor_info.get("resource_id") is not None)
assert (encl_sensor_info.get("resource_type") is not None)
assert (encl_sensor_info.get("event_time") is not None)
assert (encl_sensor_info.get("description") is not None)
encl_specific_info = encl_sensor_msg.get("specific_info")
if encl_specific_info:
assert (encl_specific_info.get("event") is not None)
def test_host_update_data_sensor(args):
check_sspl_ll_is_running()
node_data_sensor_message_request("node:os:memory_usage")
host_update_msg = None
sleep(10)
while not world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
RabbitMQingressProcessorTests.name()]._read_my_msgQ()
sleep(0.1)
print("Received for host_data: {0}".format(ingressMsg))
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("sensor_response_type")
if msg_type.get("info").get(
"resource_type") == "node:os:memory_usage":
host_update_msg = msg_type
break
except Exception as exception:
sleep(0.1)
print(exception)
assert (host_update_msg is not None)
assert (host_update_msg.get("alert_type") is not None)
assert (host_update_msg.get("alert_id") is not None)
assert (host_update_msg.get("severity") is not None)
assert (host_update_msg.get("host_id") is not None)
assert (host_update_msg.get("info") is not None)
assert (host_update_msg.get("specific_info") is not None)
info = host_update_msg.get("info")
assert (info.get("site_id") is not None)
assert (info.get("node_id") is not None)
assert (info.get("cluster_id") is not None)
assert (info.get("rack_id") is not None)
assert (info.get("resource_type") is not None)
assert (info.get("event_time") is not None)
assert (info.get("resource_id") is not None)
assert (info.get("description") is not None)
specific_info = host_update_msg.get("specific_info")
assert (specific_info.get("loggedInUsers") is not None)
assert (specific_info.get("totalMemory") is not None)
assert (specific_info.get("runningProcessCount") is not None)
assert (specific_info.get("uname") is not None)
assert (specific_info.get("bootTime") is not None)
assert (specific_info.get("processCount") is not None)
assert (specific_info.get("localtime") is not None)
def test_node_psu_module_actuator(agrs):
check_sspl_ll_is_running()
psu_actuator_message_request("NDHW:node:fru:psu", "*")
psu_module_actuator_msg = None
time.sleep(6)
ingressMsg = {}
while not world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
RabbitMQingressProcessorTests.name()]._read_my_msgQ()
time.sleep(0.1)
print("Received: %s " % ingressMsg)
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("actuator_response_type")
if msg_type["info"]["resource_type"] == "node:fru:psu":
psu_module_actuator_msg = msg_type
break
except Exception as exception:
time.sleep(0.1)
print(exception)
assert (ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID)
assert (psu_module_actuator_msg is not None)
assert (psu_module_actuator_msg.get("alert_type") is not None)
assert (psu_module_actuator_msg.get("severity") is not None)
assert (psu_module_actuator_msg.get("host_id") is not None)
assert (psu_module_actuator_msg.get("info") is not None)
assert (psu_module_actuator_msg.get("instance_id") is not None)
psu_module_info = psu_module_actuator_msg.get("info")
assert (psu_module_info.get("site_id") is not None)
assert (psu_module_info.get("node_id") is not None)
assert (psu_module_info.get("rack_id") is not None)
assert (psu_module_info.get("resource_type") is not None)
assert (psu_module_info.get("event_time") is not None)
assert (psu_module_info.get("resource_id") is not None)
fru_specific_infos = psu_module_actuator_msg.get("specific_info", {})
if fru_specific_infos:
for fru_specific_info in fru_specific_infos:
resource_id = fru_specific_info.get("resource_id")
assert (fru_specific_info.get("States Asserted") is not None)
assert (fru_specific_info.get("Sensor Type (Discrete)")
is not None)
assert (fru_specific_info.get("resource_id") is not None)
def test_if_data_sensor(args):
check_sspl_ll_is_running()
node_data_sensor_message_request("node:interface:nw")
if_data_msg = None
#create dummy interface to get network alerts
mock_eth_interface.shuffle_nw_interface()
sleep(10)
while not world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
RabbitMQingressProcessorTests.name()]._read_my_msgQ()
sleep(0.1)
print("Received for if_data: {0}".format(ingressMsg))
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("sensor_response_type")
if msg_type["info"]["resource_type"] == "node:interface:nw":
if_data_msg = msg_type
break
except Exception as exception:
sleep(0.1)
print(exception)
assert (if_data_msg is not None)
assert (if_data_msg.get("alert_type") is not None)
assert (if_data_msg.get("alert_id") is not None)
assert (if_data_msg.get("severity") is not None)
assert (if_data_msg.get("host_id") is not None)
assert (if_data_msg.get("info") is not None)
assert (if_data_msg.get("specific_info") is not None)
if_data_info = if_data_msg.get("info")
assert (if_data_info.get("site_id") is not None)
assert (if_data_info.get("node_id") is not None)
assert (if_data_info.get("cluster_id") is not None)
assert (if_data_info.get("rack_id") is not None)
assert (if_data_info.get("resource_type") is not None)
assert (if_data_info.get("event_time") is not None)
assert (if_data_info.get("resource_id") is not None)
assert (if_data_info.get("description") is not None)
if_data_specific_info = if_data_msg.get("specific_info")
assert (if_data_specific_info is not None)
assert (if_data_specific_info.get("localtime") is not None)
assert (if_data_specific_info.get("interfaces") is not None)
def test_raid_integrity_sensor(args):
platform = check_os_platform()
if platform == "vm":
return
check_sspl_ll_is_running()
node_data_sensor_message_request("node:os:raid_integrity")
raid_data_msg = None
sleep(10)
while not world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[
RabbitMQingressProcessorTests.name()]._read_my_msgQ()
sleep(0.1)
print("Received for raid_integrity_data: {0}".format(ingressMsg))
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("sensor_response_type")
if msg_type.get("info").get(
"resource_type") == "node:os:raid_integrity":
raid_data_msg = msg_type
break
except Exception as exception:
sleep(0.1)
print(exception)
assert (raid_data_msg is not None)
assert (raid_data_msg.get("alert_type") is not None)
assert (raid_data_msg.get("alert_id") is not None)
assert (raid_data_msg.get("severity") is not None)
assert (raid_data_msg.get("host_id") is not None)
assert (raid_data_msg.get("info") is not None)
assert (raid_data_msg.get("specific_info") is not None)
info = raid_data_msg.get("info")
assert (info.get("site_id") is not None)
assert (info.get("node_id") is not None)
assert (info.get("cluster_id") is not None)
assert (info.get("rack_id") is not None)
assert (info.get("resource_type") is not None)
assert (info.get("event_time") is not None)
assert (info.get("resource_id") is not None)
assert (info.get("description") is not None)
specific_info = raid_data_msg.get("specific_info")
assert (specific_info.get("error") is None)
def test_self_hw_node_sel_event(args):
check_sspl_ll_is_running()
# fetch a good resource fron ipmitool
result = run_cmd('ipmitool sdr type Fan')
test_resource = None
if result:
for resource in result:
if 'ok' in resource.decode().lower():
# this is the first ok resource, use it
test_resource = resource.decode().split('|')[0].strip()
break
# inject event into sel list and wait for alert
print(f"Using test resource {test_resource}")
run_cmd(f"ipmitool event '{test_resource}' lcr")
# wait for fault alert
start_time = time.time()
asserted = False
while time.time() - start_time < 60: # wait for 60 seconds
if wait_for_asserted_event():
asserted = True
break
if not asserted:
print("Did not get asserted event alert.")
assert (False)
# revert the event
run_cmd(f"ipmitool event '{test_resource}' lcr deassert")
# wait for alert
start_time = time.time()
deasserted = False
while time.time() - start_time < 60: # wait for 60 seconds
if wait_for_deasserted_event():
deasserted = True
break
if not deasserted:
print("Did not get asserted event alert.")
assert (False)
else:
print("ipmitool returned no results.")
assert (False)
def test_node_fan_module_actuator(agrs):
check_sspl_ll_is_running()
fan_actuator_message_request("NDHW:node:fru:fan", str(test_resource))
fan_module_actuator_msg = None
time.sleep(6)
ingressMsg = {}
while not world.sspl_modules[RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
ingressMsg = world.sspl_modules[RabbitMQingressProcessorTests.name()]._read_my_msgQ()
time.sleep(0.1)
print("Received: %s" % ingressMsg)
try:
# Make sure we get back the message type that matches the request
msg_type = ingressMsg.get("actuator_response_type")
if msg_type["info"]["resource_type"] == "node:fru:fan":
fan_module_actuator_msg = msg_type
break
except Exception as exception:
time.sleep(0.1)
print(exception)
assert(ingressMsg.get("sspl_ll_msg_header").get("uuid") == UUID)
assert(fan_module_actuator_msg is not None)
assert(fan_module_actuator_msg.get("alert_type") is not None)
assert(fan_module_actuator_msg.get("severity") is not None)
assert(fan_module_actuator_msg.get("host_id") is not None)
assert(fan_module_actuator_msg.get("info") is not None)
assert(fan_module_actuator_msg.get("instance_id") is not None)
fan_module_info = fan_module_actuator_msg.get("info")
assert(fan_module_info.get("site_id") is not None)
assert(fan_module_info.get("node_id") is not None)
assert(fan_module_info.get("rack_id") is not None)
assert(fan_module_info.get("resource_type") is not None)
assert(fan_module_info.get("event_time") is not None)
assert(fan_module_info.get("resource_id") is not None)
fru_specific_infos = fan_module_actuator_msg.get("specific_info", {})
if fru_specific_infos and fan_module_info.get("resource_id") == "*":
for fru_specific_info in fru_specific_infos:
resource_id = fru_specific_info.get("resource_id")
if "Fan Fail" in resource_id:
assert(fru_specific_info.get("Sensor Type (Discrete)") is not None)
assert(fru_specific_info.get("resource_id") is not None)
elif "System Fan" in resource_id:
assert(fru_specific_info.get("Status") is not None)
assert(fru_specific_info.get("Sensor Type (Threshold)") is not None)
assert(fru_specific_info.get("Sensor Reading") is not None)
assert(fru_specific_info.get("Lower Non-Recoverable") is not None)
assert(fru_specific_info.get("Assertions Enabled") is not None)
assert(fru_specific_info.get("Upper Non-Critical") is not None)
assert(fru_specific_info.get("Upper Non-Recoverable") is not None)
assert(fru_specific_info.get("Positive Hysteresis") is not None)
assert(fru_specific_info.get("Lower Critical") is not None)
assert(fru_specific_info.get("Deassertions Enabled") is not None)
assert(fru_specific_info.get("Lower Non-Critical") is not None)
assert(fru_specific_info.get("Upper Critical") is not None)
assert(fru_specific_info.get("Negative Hysteresis") is not None)
assert(fru_specific_info.get("Assertion Events") is not None)
assert(fru_specific_info.get("resource_id") is not None)
else:
assert(fru_specific_info.get("States Asserted") is not None)
assert(fru_specific_info.get("Sensor Type (Discrete)") is not None)
assert(fru_specific_info.get("resource_id") is not None)
elif fru_specific_infos:
assert(fru_specific_infos.get("Sensor Type (Threshold)") is not None)
assert(fru_specific_infos.get("Sensor Reading") is not None)
assert(fru_specific_infos.get("Status") is not None)
assert(fru_specific_infos.get("Lower Non_Recoverable") is not None)
assert(fru_specific_infos.get("Lower Critical") is not None)
assert(fru_specific_infos.get("Lower Non_Critical") is not None)
assert(fru_specific_infos.get("Upper Non_Critical") is not None)
assert(fru_specific_infos.get("Upper Critical") is not None)
assert(fru_specific_infos.get("Upper Non_Recoverable") is not None)
assert(fru_specific_infos.get("Positive Hysteresis") is not None)
assert(fru_specific_infos.get("Negative Hysteresis") is not None)
assert(fru_specific_infos.get("Assertion Events") is not None)
assert(fru_specific_infos.get("Assertions Enabled") is not None)
assert(fru_specific_infos.get("Deassertions Enabled") is not None)
assert(fru_specific_infos.get("resource_id") is not None)
