def checkRedfishURIs(bmcName): dbgPrint(dbgMed, "checkRedfishURIs") hostPath = "https://" + bmcName badResults = 0 for entry in URIData: path = hostPath + entry[URI] dbgPrint(dbgMed, "checkRedfishURIs checking " + path) payload, label, msg = makeRedfishCall("GET", path) if not payload: printError("checkRedfishURIs") printExtraError(label, msg) badResults += 1 continue response = json.loads(payload) idx = 1 while idx < len(entry): e = entry[idx] badResults += validateField("checkRedfishURIs", entry[URI], e[FIELD], response, e[TYPE]) idx += 1 if badResults == 0: printOK("checkRedfishURIs") return badResults
def checkRedfishEventService(bmcName): dbgPrint(dbgMed, "checkRedfishEventService") badResults = 0 path = "https://" + bmcName + "/redfish/v1/EventService" dbgPrint(dbgMed, "checkRedfishEventService checking " + path) payload, label, msg = makeRedfishCall("GET", path) if not payload: printError("checkRedfishEventService") printExtraError(label, msg) return 1 response = json.loads(payload) checkURIs = [] if response["@odata.type"] < "#EventService.v1_3_0.EventService": checkURIs = eventServiceURIs_pre_1_3 else: checkURIs = eventServiceURIs_1_3 for check in checkURIs: badResults += validateField("checkRedfishEventService", "/redfish/v1/EventService", check[FIELD], response, check[TYPE]) if badResults == 0: printOK("checkRedfishEventService") return badResults
def checkRedfishUpdateService(bmcName): fname = "checkRedfishUpdateService" dbgPrint(dbgMed, fname) badResults = 0 path = "https://" + bmcName + "/redfish/v1/UpdateService" dbgPrint(dbgMed, fname + " checking " + path) payload, label, msg = makeRedfishCall("GET", path) if not payload: printError(fname) printExtraError(label, msg) return 1 response = json.loads(payload) # Check Actions structure aField = response["Actions"]["#UpdateService.SimpleUpdate"] for e in actionFields: badResults += validateField(fname, "/redfish/v1/UpdateService", e[FIELD], aField, e[TYPE]) badResults += checkRedfishFirmwareInventory(bmcName, response["FirmwareInventory"]["@odata.id"]) if badResults == 0: printOK(fname) return badResults
def get_xname_status(xname, auth_token): dbgPrint(dbgMed, "get_xname_status") postHeaders = { 'Authorization': 'Bearer %s' % auth_token, 'cache-control': 'no-cache', 'Content-Type': 'application/json', } payload = { 'xnames': [xname], } URL = "https://api-gw-service-nmn.local/apis/capmc/capmc/v1/get_xname_status" dbgPrint(dbgMed, "POST: %s %s %s" % (URL, postHeaders, payload)) r = requests.post(url = URL, headers = postHeaders, data = json.dumps(payload)) dbgPrint(dbgMed, "Response: %s" % r.text) label = "" msg = "" if r.status_code >= 500: label = "CAPMC" msg = "Internal CAPMC Error" elif r.status_code >= 400: label = xname msg = "Bad Request" elif r.status_code >= 300: label = "CAPMC" msg = "URI redirection" if r.status_code >= 300: printError("get_xname_status") printExtraError(label, msg) return 1 status = json.loads(r.text) err = status['e'] if err < 0: printError("get_xname_status") printExtraError(xname, "Could not talk to BMC, undefined") return 1 printOK("get_xname_status") return 0
def eventDelete(bmcName): dbgPrint(dbgMed, "eventDelete") ipAddr = getIPAddress() if not ipAddr: printError("eventDelete") printExtraError("vlan004", "could not determine IP addr") return 1 path = "https://%s/redfish/v1/EventService/Subscriptions" % bmcName payload, label, msg = makeRedfishCall("GET", path) if not payload: printError("eventDelete") printExtraError(label, msg) return 1 subCollection = json.loads(payload) count = 0 for subEntry in subCollection['Members']: path = "https://%s%s" % (bmcName, subEntry['@odata.id']) payload, label, msg = makeRedfishCall("GET", path) if not payload: printError("eventDelete") printExtraError(label, msg) return 1 sub = json.loads(payload) if (sub['Context'] == "RFSubTest-%s-RFSubTest" % bmcName and sub['Destination'] == "https://%s/receiver" % ipAddr): count += 1 payload, label, msg = makeRedfishCall("DELETE", path) if not payload: printError("eventDelete") printExtraError(label, msg) return 1 dbgPrint(dbgMed, "%d subscriptions deleted for %s" % (count, bmcName)) printOK("eventDelete") return 0
def checkRedfishChassis(bmcName): dbgPrint(dbgMed, "checkRedfishChassis") badResults = 0 path = "https://" + bmcName + "/redfish/v1/Chassis" dbgPrint(dbgMed, "checkRedfishChassis checking " + path) payload, label, msg = makeRedfishCall("GET", path) if not payload: printError("checkRedfishChassis") printExtraError(label, msg) return 1 response = json.loads(payload) if "Members" not in response: printError("checkRedfishChassis") printExtraError(path + " .Members", "missing") return 1 for member in response["Members"]: path = "https://" + bmcName + member["@odata.id"] dbgPrint(dbgMed, "checkRedfishChassis checking " + path) payload, label, msg = makeRedfishCall("GET", path) if not payload: printError("checkRedfishChassis") printExtraError(label, msg) badResults += 1 continue mResponse = json.loads(payload) if ("ChassisType" in mResponse and (mResponse["ChassisType"] == "Enclosure" or mResponse["ChassisType"] == "RackMount")): for check in chassisURIs: badResults += validateField("checkRedfishChassis", member["@odata.id"], check[FIELD], mResponse, check[TYPE]) else: printInfo("checkRedfishChassis") printExtraInfo("Skipping " + member["@odata.id"], "URI is for a " + mResponse["ChassisType"]) if badResults == 0: printOK("checkRedfishChassis") return badResults
def checkRedfishSystems(bmcName): fname = "checkRedfishSystems" dbgPrint(dbgMed, fname) badResults = 0 path = "https://" + bmcName + "/redfish/v1/Systems" dbgPrint(dbgMed, fname + " checking " + path) payload, label, msg = makeRedfishCall("GET", path) if not payload: printError(fname) printExtraError(label, msg) return 1 response = json.loads(payload) if "Members" not in response: printError(fname) printExtraError(path + " .Members", "missing") return 1 for member in response["Members"]: path = "https://" + bmcName + member["@odata.id"] dbgPrint(dbgMed, fname + " checking " + path) payload, label, msg = makeRedfishCall("GET", path) if not payload: printError(fname) printExtraError(label, msg) badResults += 1 continue mResponse = json.loads(payload) for check in systemsURIs: badResults += validateField(fname, member["@odata.id"], check[FIELD], mResponse, check[TYPE]) if check[FIELD] == "Memory" and check[FIELD] in mResponse: badResults += checkRedfishSystemsMemory( bmcName, mResponse[check[FIELD]]["@odata.id"]) if check[FIELD] == "Processors" and check[FIELD] in mResponse: badResults += checkRedfishSystemsProcessors( bmcName, mResponse[check[FIELD]]["@odata.id"]) if badResults == 0: printOK(fname) return badResults
def eventSubscribe(bmcName): dbgPrint(dbgMed, "eventSubscribe") ipAddr = getIPAddress() if not ipAddr: printError("eventSubscribe") printExtraError("vlan004", "could not determine IP addr") return 1 path = "https://" + bmcName + "/redfish/v1/EventService" dbgPrint(dbgMed, "checkRedfishEventService checking " + path) payload, label, msg = makeRedfishCall("GET", path) if not payload: printError("checkRedfishEventService") printExtraError(label, msg) return 1 response = json.loads(payload) sub = { 'Context': "RFSubTest-%s-RFSubTest" % bmcName, 'Destination': "https://%s/receiver" % ipAddr, 'Protocol': 'Redfish', } if response["@odata.type"] < "#EventService.v1_3_0.EventService": eventTypes = ["StatusChange"] sub['EventTypes'] = eventTypes path = "https://%s/redfish/v1/EventService/Subscriptions" % bmcName payload, label, msg = makeRedfishCall("POST", path, json.dumps(sub)) if not payload: printError("eventSubscribe") printExtraError(label, msg) return 1 printOK("eventSubscribe") return 0
def checkRedfishManagers(bmcName): dbgPrint(dbgMed, "checkRedfishManagers") badResults = 0 path = "https://" + bmcName + "/redfish/v1/Managers" dbgPrint(dbgMed, "checkRedfishManagers checking " + path) payload, label, msg = makeRedfishCall("GET", path) if not payload: printError("checkRedfishManagers") printExtraError(label, msg) return 1 response = json.loads(payload) if "Members" not in response: printError("checkRedfishManagers") printExtraError(path + " .Members", "missing") return 1 for member in response["Members"]: path = "https://" + bmcName + member["@odata.id"] dbgPrint(dbgMed, "checkRedfishManagers checking " + path) payload, label, msg = makeRedfishCall("GET", path) if not payload: printError("checkRedfishManagers") printExtraError(label, msg) badResults += 1 continue mResponse = json.loads(payload) for check in managerURIs: badResults += validateField("checkRedfishManagers", member["@odata.id"], check[FIELD], mResponse, check[TYPE]) if badResults == 0: printOK("checkRedfishManagers") return badResults
def get_power_cap_capabilities(xname, auth_token): dbgPrint(dbgMed, "get_power_cap_capabilities") global capMin global capMax nid = getNid(xname, auth_token) if nid < 0: printError("get_power_cap_capabilities") printExtraError(xname, "Could not get nid") return 1 postHeaders = { 'Authorization': 'Bearer %s' % auth_token, 'cache-control': 'no-cache', 'Content-Type': 'application/json', } payload = { 'nids': [nid], } URL = "https://api-gw-service-nmn.local/apis/capmc/capmc/v1/get_power_cap_capabilities" dbgPrint(dbgMed, "POST: %s %s %s" % (URL, postHeaders, payload)) r = requests.post(url = URL, headers = postHeaders, data = json.dumps(payload)) dbgPrint(dbgMed, "Response: %s" % r.text) if r.status_code >= 500: label = "CAPMC" msg = "Internal CAPMC Error" elif r.status_code >= 400: label = xname msg = "Bad Request" elif r.status_code >= 300: label = "CAPMC" msg = "URI redirection" if r.status_code >= 300: printError("get_power_cap_capabilities") printExtraError(label, msg) return 1 capInfo = json.loads(r.text) group = capInfo['groups'][0] if group['controls']: control = None for tmp in group['controls']: if tmp['name'].startswith('Node'): control = tmp break if control != None: capMax = control['max'] capMin = control['min'] supply = group['supply'] if capMax == 0: if supply == 0: printError("get_power_cap_capabilities") printExtraError("min", capMin) printExtraError("max", capMax) printExtraError("supply", supply) return 1 else: capMax = supply printOK("get_power_cap_capabilities") return 0
def get_node_energy_counter(xname, auth_token): dbgPrint(dbgMed, "get_node_energy_counter") nid = getNid(xname, auth_token) if nid < 0: printError("get_node_energy_stats") printExtraError(xname, "Could not get nid") return 1 postHeaders = { 'Authorization': 'Bearer %s' % auth_token, 'cache-control': 'no-cache', 'Content-Type': 'application/json', } etime = datetime.today() stime = etime - timedelta(hours=1) payload = { 'nids': [nid], 'start_time': stime.strftime('%Y-%m-%d %H:%M:%S'), 'end_time': etime.strftime('%Y-%m-%d %H:%M:%S'), } URL = "https://api-gw-service-nmn.local/apis/capmc/capmc/v1/get_node_energy_counter" dbgPrint(dbgMed, "POST: %s %s %s" % (URL, postHeaders, payload)) r = requests.post(url = URL, headers = postHeaders, data = json.dumps(payload)) dbgPrint(dbgMed, "Response: %s" % r.text) label = "" msg = "" if r.status_code >= 500: label = "CAPMC" msg = "Internal CAPMC Error" elif r.status_code >= 400: label = xname msg = "Bad Request" elif r.status_code >= 300: label = "CAPMC" msg = "URI redirection" if r.status_code >= 300: printError("get_node_energy_counter") printExtraError(label, msg) return 1 energyInfo = json.loads(r.text) err = energyInfo['e'] if err > 0: printError("get_node_energy_counter") printExtraError(xname, "No data in time window") return 1 energy = energyInfo['nodes'][0]['energy_ctr'] if energy <= 0: printError("get_node_energy_counter") printExtraError("energy_ctr", energy) return 1 printOK("get_node_energy_counter") return 0
def set_power_cap(xname, auth_token): dbgPrint(dbgMed, "set_power_cap") if capMax == 0: printError("set_power_cap") printExtraError("Invalid max cap value", capMax) return 1 nid = getNid(xname, auth_token) if nid < 0: printError("set_power_cap") printExtraError(xname, "Could not get nid") return 1 # Get and save original value postHeaders = { 'Authorization': 'Bearer %s' % auth_token, 'cache-control': 'no-cache', 'Content-Type': 'application/json', } payload = { 'nids': [nid], } URL = "https://api-gw-service-nmn.local/apis/capmc/capmc/v1/get_power_cap" dbgPrint(dbgMed, "POST: %s %s %s" % (URL, postHeaders, payload)) r = requests.post(url = URL, headers = postHeaders, data = json.dumps(payload)) dbgPrint(dbgMed, "Response: %s" % r.text) if r.status_code >= 500: label = "CAPMC" msg = "Internal CAPMC Error" elif r.status_code >= 400: label = xname msg = "Bad Request" elif r.status_code >= 300: label = "CAPMC" msg = "URI redirection" if r.status_code >= 300: printError("set_power_cap") printExtraError(label, msg) return 1 capInfo = json.loads(r.text) err = capInfo['e'] if err != 0: printError("set_power_cap") printExtraError(xname, "Node not in the Ready state") return 1 origVal = extract_power_cap_val(capInfo) if origVal is not None and origVal <= 0: printError("set_power_cap") printExtraError("value", origVal) return 1 # Verify in range if origVal is not None and (origVal < capMin or origVal > capMax): printError("set_power_cap") printExtraError("value", origVal) return 1 # Set to (max - 10) postHeaders = { 'Authorization': 'Bearer %s' % auth_token, 'cache-control': 'no-cache', 'Content-Type': 'application/json', } capVal = capMax - 10 payload = { 'nids': [{'controls': [{'name': 'node', 'val': capVal}], 'nid': nid}], } URL = "https://api-gw-service-nmn.local/apis/capmc/capmc/v1/set_power_cap" dbgPrint(dbgMed, "POST: %s %s %s" % (URL, postHeaders, payload)) r = requests.post(url = URL, headers = postHeaders, data = json.dumps(payload)) dbgPrint(dbgMed, "Response: %s" % r.text) if r.status_code >= 500: label = "CAPMC" msg = "Internal CAPMC Error" elif r.status_code >= 400: label = xname msg = "Bad Request" elif r.status_code >= 300: label = "CAPMC" msg = "URI redirection" if r.status_code >= 300: printError("set_power_cap") printExtraError(label, msg) return 1 capInfo = json.loads(r.text) err = capInfo['e'] if err != 0: printError("set_power_cap") printExtraError(xname, capInfo['nids'][0]['err_msg']) return 1 # Get postHeaders = { 'Authorization': 'Bearer %s' % auth_token, 'cache-control': 'no-cache', 'Content-Type': 'application/json', } payload = { 'nids': [nid], } URL = "https://api-gw-service-nmn.local/apis/capmc/capmc/v1/get_power_cap" dbgPrint(dbgMed, "POST: %s %s %s" % (URL, postHeaders, payload)) r = requests.post(url = URL, headers = postHeaders, data = json.dumps(payload)) dbgPrint(dbgMed, "Response: %s" % r.text) if r.status_code >= 500: label = "CAPMC" msg = "Internal CAPMC Error" elif r.status_code >= 400: label = xname msg = "Bad Request" elif r.status_code >= 300: label = "CAPMC" msg = "URI redirection" if r.status_code >= 300: printError("set_power_cap") printExtraError(label, msg) return 1 capInfo = json.loads(r.text) err = capInfo['e'] if err != 0: printError("set_power_cap") printExtraError(xname, "Node not in the Ready state") return 1 setVal = extract_power_cap_val(capInfo) if setVal is not None and setVal <= 0: printError("set_power_cap") printExtraError("value", setVal) return 1 # Verify avg of min/max if setVal != capVal: printError("set_power_cap") printExtraError("set value", setVal) printExtraError("expected value", capVal) return 1 # Set original value postHeaders = { 'Authorization': 'Bearer %s' % auth_token, 'cache-control': 'no-cache', 'Content-Type': 'application/json', } if origVal is None: capVal = 0 else: capVal = origVal payload = { 'nids': [{'controls': [{'name': 'node', 'val': capVal}], 'nid': nid}], } URL = "https://api-gw-service-nmn.local/apis/capmc/capmc/v1/set_power_cap" dbgPrint(dbgMed, "POST: %s %s %s" % (URL, postHeaders, payload)) r = requests.post(url = URL, headers = postHeaders, data = json.dumps(payload)) dbgPrint(dbgMed, "Response: %s" % r.text) if r.status_code >= 500: label = "CAPMC" msg = "Internal CAPMC Error" elif r.status_code >= 400: label = xname msg = "Bad Request" elif r.status_code >= 300: label = "CAPMC" msg = "URI redirection" if r.status_code >= 300: printError("set_power_cap") printExtraError(label, msg) return 1 printOK("set_power_cap") return 0
def get_power_cap(xname, auth_token): dbgPrint(dbgMed, "get_power_cap") nid = getNid(xname, auth_token) if nid < 0: printError("get_power_cap") printExtraError(xname, "Could not get nid") return 1 postHeaders = { 'Authorization': 'Bearer %s' % auth_token, 'cache-control': 'no-cache', 'Content-Type': 'application/json', } payload = { 'nids': [nid], } URL = "https://api-gw-service-nmn.local/apis/capmc/capmc/v1/get_power_cap" dbgPrint(dbgMed, "POST: %s %s %s" % (URL, postHeaders, payload)) r = requests.post(url = URL, headers = postHeaders, data = json.dumps(payload)) dbgPrint(dbgMed, "Response: %s" % r.text) if r.status_code >= 500: label = "CAPMC" msg = "Internal CAPMC Error" elif r.status_code >= 400: label = xname msg = "Bad Request" elif r.status_code >= 300: label = "CAPMC" msg = "URI redirection" if r.status_code >= 300: printError("get_power_cap") printExtraError(label, msg) return 1 capInfo = json.loads(r.text) err = capInfo['e'] if err != 0: printError("get_power_cap") printExtraError(xname, capInfo['nids'][0]['err_msg']) return 1 val = capInfo['nids'][0]['controls'][0]['val'] if val is not None and val <= 0: printError("get_power_cap") printExtraError("value", val) return 1 printOK("get_power_cap") return 0
def eventTest(bmcName): dbgPrint(dbgMed, "eventTest") global event hostPath = "https://" + bmcName path = hostPath + "/redfish/v1/Chassis" payload, label, msg = makeRedfishCall("GET", path) if not payload: printError("telemetryPoll") printExtraError(label, msg) return 1 chassisList = json.loads(payload) chassis = chassisList['Members'][0] testEvent = {} path = hostPath + "/redfish/v1/EventService/Actions/EventService.SubmitTestEvent" if isGigabyte(chassis['@odata.id']): evID = randrange(10000, 99999) testEvent = { "EventTimestamp": "2020-05-26T23:04:09+02:00", "EventId": evID, "OriginOfCondition": "/redfish/v1/Chassis/Self", "MessageId": "PropertyValueNotInList", "MessageArgs": ["Lit", "IndicatorLED"], "Severity": "Warning" } elif isHPERiver(chassis['@odata.id']): testEvent = { 'EventID': 'Test Event', 'Severity': 'OK', 'EventType': 'StatusChange', 'OriginOfCondition': 'Test', 'EventTimestamp': str(datetime.datetime.now()), 'MessageArgs': [], 'Message': 'This is a test event', 'MessageId': 'TestMsg.v0' } elif isHPEMountain(chassis['@odata.id']): printInfo("eventValidate") printExtraInfo("Test Event", "Not supported") return 1 if httpd is None: startRedfishEventServer() payload, label, msg = makeRedfishCall("POST", path, json.dumps(testEvent)) if not payload: printError("eventValidate") printExtraError(label, msg) return 1 if event.wait(timeout=30): printOK("eventValidate") return 0 else: printError("eventValidate") printExtraError("event", "timed out waiting for Redfish test event") return 1
def telemetryPoll(bmcName): dbgPrint(dbgMed, "telemetryPoll") hostPath = "https://" + bmcName path = hostPath + "/redfish/v1/Chassis" payload, label, msg = makeRedfishCall("GET", path) if not payload: printError("telemetryPoll") printExtraError(label, msg) return 1 chassisList = json.loads(payload) badResults = 0 for chassis in chassisList['Members']: chassisPath = hostPath + chassis['@odata.id'] path = chassisPath + "/Power" payload, label, msg = makeRedfishCall("GET", path) if not payload: printError("telemetryPoll power") printExtraError(label, msg) return 1 Power = json.loads(payload) path = chassisPath + "/Thermal" payload, label, msg = makeRedfishCall("GET", path) if not payload: printError("telemetryPoll thermal") printExtraError(label, msg) return 1 Thermal = json.loads(payload) # Gigabyte # /Power # .PowerControl.PowerMetrics.AverageConsumedWatts # .Voltages[].ReadingVolts # /Thermal # .Fans[].Reading # .Temperatures[].ReadingCelsius if isGigabyte(chassis['@odata.id']): ret = checkAvgConsumedWatts(Power) if ret != 0: printError("telemetryPoll power") printExtraError(bmcName, "AverageConsumedWatts missing") badResults += 1 ret = checkVoltages(Power) if ret != 0: printError("telemetryPoll power") printExtraError(bmcName, "Voltages missing") badResults += 1 ret = checkFans(Thermal) if ret != 0: printError("telemetryPoll thermal") printExtraError(bmcName, "Fans missing") badResults += 1 ret = checkTemps(Thermal) if ret != 0: printError("telemetryPoll thermal") printExtraError(bmcName, "Temperatures missing") badResults += 1 # HPE - Mountain (we don't poll, but check what we can) # /Power # .Voltages[].ReadingVolts # /Thermal # .Temperatures[].ReadingCelsius if isHPEMountain(chassis['@odata.id']): ret = checkVoltages(Power) if ret != 0: printError("telemetryPoll power") printExtraError(bmcName, "Voltages missing") badResults += 1 ret = checkTemps(Thermal) if ret != 0: printError("telemetryPoll thermal") printExtraError(bmcName, "Temperatures missing") badResults += 1 # HPE - River # /Power # .PowerControl.PowerMetrics.AverageConsumedWatts # .PowerSupplies[].LineInputVoltage # /Thermal # .Fans[].Reading # .Temperatures[].ReadingCelsius if isHPERiver(chassis['@odata.id']): ret = checkAvgConsumedWatts(Power) if ret != 0: printError("telemetryPoll power") printExtraError(bmcName, "AverageConsumedWatts missing") badResults += 1 ret = checkLineVoltages(Power) if ret != 0: printError("telemetryPoll power") printExtraError(bmcName, "Voltages missing") badResults += 1 ret = checkFans(Thermal) if ret != 0: printError("telemetryPoll thermal") printExtraError(bmcName, "Fans missing") badResults += 1 ret = checkTemps(Thermal) if ret != 0: printError("telemetryPoll thermal") printExtraError(bmcName, "Temperatures missing") badResults += 1 if badResults == 0: printOK("telemetryPoll") return badResults