def inspect_nodes(conn, target_stage="inspect_1GbE", initial_stage=None): nodes = ironic_drac_settings.get_nodes_in_rack(conn, "DR06") inspecting = [] for node in nodes: if node["provision_state"] in ["inspect wait", "inspecting"]: print("inspecting: " + node.name + " " + node["driver_info"]["drac_address"]) inspecting.append(node) continue # Skip node if already bootstrapped if "bootstrap_stage" in node["extra"] \ and node["extra"]["bootstrap_stage"] in [target_stage]: print("Stage invalid, exiting") continue # Skip node if already bootstrapped if "bootstrap_stage" not in node["extra"] \ or node["extra"]["bootstrap_stage"] not in [initial_stage]: print("Not ready for inspect stage") continue if node["provision_state"] != "manageable": print("Ignoring node, invalid state") continue if node["is_maintenance"]: print("skip nodes in maintenance") continue if node["power_state"] == "power on": conn.baremetal.set_node_power_state(node, "power off") extra = node["extra"] extra["bootstrap_stage"] = target_stage patch = [ { "op": "replace", "path": "inspect_interface", "value": "inspector" }, { "op": "replace", "path": "extra", "value": extra }] conn.baremetal.patch_node(node, patch) conn.baremetal.set_node_provision_state(node, 'inspect') inspecting.append(node) print("inspecting: " + node.name + " " + node["driver_info"]["drac_address"]) time.sleep(2) conn.baremetal.wait_for_nodes_provision_state(inspecting, 'manageable')
def boot_on_cleaning_net(conn): nodes = ironic_drac_settings.get_nodes_in_rack(conn, "DR06") nodes = nodes[1:2] print(nodes[0]) exit(-1) clients = {} for node in nodes: if node["provision_state"] != "manageable": print("Ignoring node, invalid state") continue manual_setup_port_inspection(conn, node)
def move_back_to_manageable(conn): nodes = ironic_drac_settings.get_nodes_in_rack(conn, "DR06") pending = [] for node in nodes: if node["provision_state"] != "available": print("Ignoring node, invalid state") continue print("aborting clean: " + node.name + " " + node["driver_info"]["drac_address"]) conn.baremetal.set_node_provision_state(node, 'manage') pending.append(node) conn.baremetal.wait_for_nodes_provision_state(pending, 'manageable')
def test_inspector_pxe_boot(conn): # Server to operate on #id = sys.argv[1] #node = conn.baremetal.find_node(id) #if not node: # print("Bailing out: node not found") # sys.exit(1) nodes = ironic_drac_settings.get_nodes_in_rack(conn, "DR06") #nodes = nodes[0:2] print(len(nodes)) pending = [] for node in nodes: # Skip node if already bootstrapped if "bootstrap_stage" in node["extra"] \ and node["extra"]["bootstrap_stage"] in ["inspect"]: print("Stage invalid, exiting") continue if node["provision_state"] != "manageable": print("Ignoring node, invalid state") continue # Setup dhcp to hand out boot port = setup_port(conn, node) print(port) # Ask for power on, if not already # TODO: better handle nodes that are already turned on? # TODO: reboot: conn.baremetal.set_node_power_state(node, "power off") if node["power_state"] != "power on": time.sleep(2) # be conservative about power demand conn.baremetal.set_node_power_state(node, "power on") print("Powered on: " + node["name"]) # tell ironic not to mess with power # as we expect to reboot a few times if not node["is_maintenance"]: # Don't power down during firmware upgrade! conn.baremetal.set_node_maintenance( node, reason="PXE to flash nic firmware") print("Waiting for: " + node["name"]) pending.append(node) time.sleep(5) check_pending(conn, pending)
def set_expected_bios_version(conn, version="2.6.3"): nodes = ironic_drac_settings.get_nodes_in_rack(conn, "DR06") for node in nodes: vendor = node["extra"].get("system_vendor") if not vendor: print(f"node in bad state: {node['name']}") elif vendor.get("bios_version") != version: print("setting bios") patch = [{ "op": "add", "path": "extra/system_vendor/bios_version", "value": version }] print(f"patching node {node['name']}") conn.baremetal.patch_node(node, patch)
def reset_after_failed_clean(conn): nodes = ironic_drac_settings.get_nodes_in_rack(conn, "DR06") pending = [] for node in nodes: if node["provision_state"] != "clean failed": print("Ignoring node, invalid state") continue print("moving to manageable: " + node.name + " " + node["driver_info"]["drac_address"]) if node["is_maintenance"]: conn.baremetal.unset_node_maintenance(node) if node["power_state"] == "power on": conn.baremetal.set_node_power_state(node, "power off") conn.baremetal.set_node_provision_state(node, 'manage') pending.append(node) conn.baremetal.wait_for_nodes_provision_state(pending, 'manageable')
def move_to_available(conn): nodes = ironic_drac_settings.get_nodes_in_rack(conn, "DR06") pending = [] for node in nodes: if node["provision_state"] in ["clean wait", "cleaning"]: print("already cleaning: " + node.name + " " + node["driver_info"]["drac_address"]) pending.append(node) continue if node["provision_state"] != "manageable": print("Ignoring node, invalid state") continue name = node["name"] # Skip node if already bootstrapped if "bootstrap_stage" in node["extra"]: if node["extra"]["bootstrap_stage"] == "boot_on_50GbE": dclient = get_dracclient(node) clients[name] = dclient continue if node["extra"]["bootstrap_stage"] not in [ "inspect_50GbE", "boot_on_50GbE_no_1G" ]: print("Stage invalid, exiting") continue #if node["extra"]["bootstrap_stage"] != "inspect_50GbE": # print("Stage invalid, exiting") # continue if node["is_maintenance"]: print("Skip nodes in maintenance") continue print("moving to available: " + node.name + " " + node["driver_info"]["drac_address"]) conn.baremetal.set_node_provision_state(node, 'provide') pending.append(node) #time.sleep(2) #if len(pending) >= 4: # break conn.baremetal.wait_for_nodes_provision_state(pending, 'available')
def request_hse_boot(conn): nodes = ironic_drac_settings.get_nodes_in_rack(conn, "DR06") clients = {} for node in nodes: if node["provision_state"] != "manageable": print("Ignoring node, invalid state") continue name = node["name"] # Skip node if already bootstrapped if "bootstrap_stage" in node["extra"]: if node["extra"]["bootstrap_stage"] == "boot_on_50GbE": dclient = get_dracclient(node) clients[name] = dclient continue if node["extra"]["bootstrap_stage"] != "inspect_1GbE": print("Stage invalid, exiting") continue #if node["extra"]["bootstrap_stage"] != "inspect_50GbE": # print("Stage invalid, exiting") # continue if node["is_maintenance"]: print("Skip nodes in maintenance") continue print("moving to hse boot: " + node.name + " " + node["driver_info"]["drac_address"]) dclient = get_dracclient(node) bios_settings = { "LogicalProc": "Disabled", "SysProfile": "PerfOptimized", "EmbNic1": "DisabledOs", #"SetBootOrderEn": "NIC.Slot.4-1,InfiniBand.Slot.4-1,NIC.Embedded.1-1-1,HardDisk.List.1-1", "SetBootOrderFqdd1": "NIC.Slot.4-1", "SetBootOrderFqdd2": "HardDisk.List.1-1", "SetBootOrderFqdd3": "", "SetBootOrderFqdd4": "", } idrac_settings={ } ironic_drac_settings.update_settings( dclient, bios_settings, idrac_settings) clients[name] = dclient extra = node["extra"] extra["bootstrap_stage"] = "boot_on_50GbE" patch = [ { "op": "replace", "path": "inspect_interface", "value": "inspector" }, { "op": "replace", "path": "extra", "value": extra }] conn.baremetal.patch_node(node, patch) # TODO: ideally add a new state for when jobs are complete ironic_drac_settings.wait_for_jobs(clients)
def get_inspection_data(conn): nodes = ironic_drac_settings.get_nodes_in_rack(conn, "DR06") result = [] for raw_node in nodes: ports = list(conn.baremetal.ports(node=raw_node["id"], details=True)) name = raw_node['name'] ip = raw_node["driver_info"]["drac_address"] node = { "name": name, "ip": ip, "uuid": raw_node['id'], "ports_by_mac": {}, "ports_by_switch": {}, "service_tag": "", "bmc_mac": "", "mac": "", "rack": "", "datacentre": "", } node["rack_pos"] = name.split("u")[1] idrac_ports = list(conn.network.ports(name=name)) if len(idrac_ports) == 1: idrac_port = idrac_ports[0] if idrac_port["fixed_ips"][0]["ip_address"] == ip: node["bmc_mac"] = idrac_port['mac_address'] node["rack"] = [tag for tag in idrac_port["tags"] if "DR" in tag][0] node["datacentre"] = [tag for tag in idrac_port["tags"] if "DC" in tag][0] conn.add_service("baremetal-introspection") inspector = conn.baremetal_introspection response = inspector.get(f"/introspection/{name}/data") if response: node["interfaces"] = list(response.json()["interfaces"].keys()) node["hse_interfaces"] = [i for i in node["interfaces"] if "p" in i] extra = raw_node['extra'] if extra and "system_vendor" in extra: node["service_tag"] = extra['system_vendor'].get('serial_number') lldp_count = 0 for raw_port in ports: lldp = raw_port['local_link_connection'] if lldp: lldp_count += 1 mac = raw_port['address'] node["ports_by_mac"][mac] = lldp if lldp and "GigabitEthernet" in lldp["port_id"]: port = lldp["port_id"] if port: port = port.split("GigabitEthernet 1/")[1] node["ports_by_switch"]["s3048"] = { "mac": mac, "host": lldp["switch_info"], "port": port, } elif lldp and "swp" in lldp["port_id"]: port = lldp["port_id"] if port: port = port.split("swp")[1] port = port.split("s")[0] node["ports_by_switch"]["sn3700"] = { "mac": mac, "host": lldp["switch_info"], "port": port, } result.append(node) #if lldp_count < 2: # result.append(node) #for node in result: # if "hse_interfaces" in node: # print(f'{node["name"]}\n{node["interfaces"]}') #exit(0) print(json.dumps(result, indent=2)) print("dc,rack,rack_pos,height,hardware_name,manufacturer,model,serial," "name,ip,mac_noformat,mac,bmc_ip,bmc_mac_noformat,bmc_mac," "nodetype,groups,s3048_port,sn3700c_port") csv = {} csv_structured = {} for node in result: name = node["name"] oob = node["ports_by_switch"].get("s3048", {}) mac = oob.get("mac", "").upper() mac_noformat = mac.replace(":","") bmc_mac = node.get("bmc_mac", "").upper() bmc_mac_noformat = bmc_mac.replace(":", "") hse = node["ports_by_switch"].get("sn3700", {}) oob_port = "" if oob: oob_port = "-p".join([oob.get("host"), oob.get("port")]) hse_port = "" if hse: hse_port = "-p".join([hse.get("host"), hse.get("port")]) groups = ( 'all,nodes,cascadelake,csd3,compute-csd3,compute-cascadelake,' f'Dell,C6420,csd3-2020q3p1,csd3-2020q3p1-{node["rack"].lower()}' ) node_csv = ( f'{node["datacentre"]},{node["rack"]},{node["rack_pos"]},1,' f'{name},Dell,C6420,{node["service_tag"]},{name},,' f'{mac_noformat},{mac},,{bmc_mac_noformat},{bmc_mac},' f'{groups},{oob_port},{hse_port}') csv[name] = node_csv csv_structured[name] = { "dc": node["datacentre"], "rack": node["rack"], "rack_pos": node["rack_pos"], "height": "1", "hardware_name": node["name"], "manufacturer": "Dell", "model": "C6420", "serial": node["service_tag"], "name": node["name"], "ip": "", "mac_noformat": mac_noformat, "mac": mac, "bmc_ip": "", "bmc_mac_noformat": bmc_mac_noformat, "bmc_mac": bmc_mac, "nodetype": "server", "groups": groups, "s3048_port": oob_port, "sn3700c_port": hse_port, } print(node_csv) import csv nodes = {} with open("DR06.csv") as csvfile: rows = csv.DictReader(csvfile) for row in rows: nodes[row["name"]] = row #print(json.dumps(nodes, indent=2)) for name, expected in nodes.items(): expected["ip"] = expected["ip"].strip() expected_str = json.dumps(expected) inspected = csv_structured.get(name, {}) inspected_str = json.dumps(inspected) has_50GbE_up = inspected.get("serial") and inspected.get("sn3700c_port") if expected_str != inspected_str: print("error: " + name) #print("exp: " + expected_str) #print("ins: " + inspected_str) print("1 expected: " + expected['s3048_port'] + " found: " + inspected['s3048_port'])