def _add_psu(key, psu_index, attr): """Add a PSU to an existing server Args: key(int): server key psu will belong to psu_index(int): psu number attr(dict): psu attributes such as power_source, power_consumption, variation & draw """ with GRAPH_REF.get_session() as session: query = [] # find the server query.append("MATCH (asset:Asset {{ key: {} }})".format(key)) # create a PSU attr["key"] = int("{}{}".format(key, psu_index)) attr["name"] = "psu" + str(psu_index) attr["type"] = "psu" props_stm = qh.get_props_stm(attr) query.append( "CREATE (psu:Asset:PSU:Component {{ {} }})".format(props_stm)) # set relationships query.append("CREATE (asset)-[:HAS_COMPONENT]->(psu)") query.append("CREATE (asset)-[:POWERED_BY]->(psu)") session.run("\n".join(query))
def create_static(key, attr, static_type="StaticAsset"): """Create Dummy static asset""" if not attr["power_consumption"]: raise KeyError("Static asset requires power_consumption") with GRAPH_REF.get_session() as session: s_attr = ["img_url", "power_consumption", "power_source"] + CREATE_SHARED_ATTR props_stm = qh.get_props_stm( {**attr, **{"type": static_type.lower(), "key": key}}, supported_attr=s_attr ) session.run("CREATE (:Asset:{} {{ {} }})".format(static_type, props_stm))
def create_outlet(key, attr): """Add outlet to the model Args: key(int): unique key to be assigned attr(dict): asset properties """ with GRAPH_REF.get_session() as session: attr["name"] = ( attr["name"] if "name" in attr and attr["name"] else "out-{}".format(key) ) s_attr = CREATE_SHARED_ATTR props_stm = qh.get_props_stm( {**attr, **{"type": "outlet", "key": key}}, supported_attr=s_attr ) session.run("CREATE (:Asset:Outlet {{ {} }})".format(props_stm))
def test_get_props_stm_filter(self): """Test if supported attr filters out attr""" attr = {"a": 1, "b": 2, "c": 3, "d": 4} s_attr = ["a", "b", "c"] formatted_stm = qh.get_props_stm(attr, supported_attr=s_attr) self.assertEqual("a: 1,b: 2,c: 3", formatted_stm)
def test_get_props_stm(self): """Test props statement used to init node attributes""" attr = {"a": 1, "b": 2, "c": 3} formatted_stm = qh.get_props_stm(attr) self.assertEqual("a: 1,b: 2,c: 3", formatted_stm)
def create_pdu( key, attr, preset_file=os.path.join(os.path.dirname(__file__), "presets/apc_pdu.json"), ): """Add PDU to the model """ preset_file = (attr["snmp_preset"] if "snmp_preset" in attr and attr["snmp_preset"] else preset_file) with open( preset_file) as preset_handler, GRAPH_REF.get_session() as session: query = [] data = json.load(preset_handler) outlet_count = data["OIDs"]["OutletCount"]["defaultValue"] attr["name"] = (attr["name"] if "name" in attr and attr["name"] else data["assetName"]) s_attr = [ "staticOidFile", "port", "host", "interface", "mask", ] + CREATE_SHARED_ATTR props_stm = qh.get_props_stm( { **attr, **data, **{ "key": key, "type": "pdu" } }, supported_attr=s_attr) # create PDU asset query.append( "CREATE (pdu:Asset:PDU:SNMPSim {{ {} }})".format(props_stm)) # Add PDU OIDS to the model for oid_key, oid_props in data["OIDs"].items(): if "serial_number" in attr and attr["serial_number"]: oid_props["defaultValue"] = attr["serial_number"] else: oid_props["defaultValue"] = qh.generate_id() if oid_key == "MAC": if "mac_address" in attr and attr["mac_address"]: oid_props["defaultValue"] = attr["mac_address"] else: oid_props["defaultValue"] = qh.generate_mac() s_attr = ["OID", "OIDName", "defaultValue", "dataType"] props_stm = qh.get_props_stm({ **oid_props, **{ "OIDName": oid_key } }, supported_attr=s_attr) query.append( "CREATE (:OID {{ {props_stm} }})<-[:HAS_OID]-(pdu)".format( props_stm=props_stm)) # Outlet-specific OIDs for oid_key, oid_props in data["outletOIDs"].items(): # For outlet state, Outlet asset will need to be created if oid_key == "OutletState": oid_desc = dict( (y, x) for x, y in oid_props["oidDesc"].items()) desc_stm = qh.get_oid_desc_stm(oid_desc) query.append( "CREATE (oidDesc:OIDDesc {{ {} }})".format(desc_stm)) for j in range(outlet_count): out_key = int("{}{}".format(key, str(j + 1))) props_stm = qh.get_props_stm({ "key": out_key, "name": "out" + str(j + 1), "type": "outlet" }) # create outlet per OID query.append( "CREATE (out{}:Asset:Outlet:Component {{ {} }})". format(out_key, props_stm)) # set outlet relationships query.append( "CREATE (out{})-[:POWERED_BY]->(pdu)".format(out_key)) query.append( "CREATE (pdu)-[:HAS_COMPONENT]->(out{})".format( out_key)) # create OID associated with outlet & pdu for oid_n, oid in enumerate(oid_props["OID"]): out_key = int("{}{}".format(key, str(j + 1))) oid = oid + "." + str(j + 1) oid_node_name = "{oid_name}{outlet_num}{oid_num}".format( oid_name=oid_key, outlet_num=j, oid_num=oid_n) props_stm = qh.get_props_stm({ "OID": oid, "OIDName": oid_key, "dataType": oid_props["dataType"], "defaultValue": oid_props["defaultValue"], }) query.append( "CREATE ({oid_node_name}:OID {{ {props_stm} }})". format(oid_node_name=oid_node_name, props_stm=props_stm)) # set the relationships (outlet powerd by state oid etc..) query.append( "CREATE (out{})-[:POWERED_BY]->({})".format( out_key, oid_node_name)) query.append( "CREATE ({})-[:HAS_STATE_DETAILS]->(oidDesc)". format(oid_node_name)) query.append("CREATE (pdu)-[:HAS_OID]->({})".format( oid_node_name)) else: oid = oid_props["OID"] props_stm = qh.get_props_stm({ "OID": oid, "OIDName": oid_key, "dataType": oid_props["dataType"], "defaultValue": oid_props["defaultValue"], }) query.append("CREATE ({}:OID {{ {} }})".format( oid_key, props_stm)) query.append("CREATE (pdu)-[:HAS_OID]->({})".format(oid_key)) # print("\n".join(query)) session.run("\n".join(query))
def create_ups( key, attr, preset_file=os.path.join(os.path.dirname(__file__), "presets/apc_ups.json"), ): """Add UPS to the system model """ preset_file = (attr["snmp_preset"] if "snmp_preset" in attr and attr["snmp_preset"] else preset_file) with open( preset_file) as preset_handler, GRAPH_REF.get_session() as session: query = [] data = json.load(preset_handler) attr["name"] = (attr["name"] if "name" in attr and attr["name"] else data["assetName"]) attr["runtime"] = json.dumps(data["modelRuntime"], sort_keys=True) s_attr = [ "staticOidFile", "port", "host", "outputPowerCapacity", "minPowerOnBatteryLevel", "fullRechargeTime", "runtime", "power_source", "power_consumption", "work_dir", "interface", "mask", ] + CREATE_SHARED_ATTR props_stm = qh.get_props_stm( { **attr, **data, **{ "key": key, "type": "ups" } }, supported_attr=s_attr) # create UPS asset query.append( "CREATE (ups:Asset:UPS:SNMPSim {{ {} }})".format(props_stm)) # add batteries (only one for now) query.append( "CREATE (bat:UPSBattery:Battery { name: 'bat1', type: 'battery' })" ) query.append("CREATE (ups)-[:HAS_BATTERY]->(bat)") query.append("CREATE (ups)-[:POWERED_BY]->(bat)") # Add UPS OIDs for oid_key, oid_props in data["OIDs"].items(): if oid_key == "SerialNumber": if "serial_number" in attr and attr["serial_number"]: oid_props["defaultValue"] = attr["serial_number"] else: oid_props["defaultValue"] = qh.generate_id() if oid_key == "MAC": if "mac_address" in attr and attr["mac_address"]: oid_props["defaultValue"] = attr["mac_address"] else: oid_props["defaultValue"] = qh.generate_mac() props = {**oid_props, **{"OIDName": oid_key}} props_stm = qh.get_props_stm( props, ["OID", "dataType", "defaultValue", "OIDName"]) query.append("CREATE ({}:OID {{ {} }})".format(oid_key, props_stm)) query.append("CREATE (ups)-[:HAS_OID]->({})".format(oid_key)) if "oidDesc" in oid_props and oid_props["oidDesc"]: oid_desc = dict( (y, x) for x, y in oid_props["oidDesc"].items()) desc_stm = qh.get_oid_desc_stm(oid_desc) query.append("CREATE ({}Desc:OIDDesc {{ {} }})".format( oid_key, desc_stm)) query.append( "CREATE ({})-[:HAS_STATE_DETAILS]->({}Desc)".format( oid_key, oid_key)) if oid_key == "PowerOff": query.append( "CREATE (ups)-[:POWERED_BY]->({})".format(oid_key)) # Set output outlets for i in range(data["numOutlets"]): props = { "name": "out" + str(i + 1), "type": "outlet", "key": int("{}{}".format(key, str(i + 1))), } props_stm = qh.get_props_stm(props) query.append( "CREATE (out{}:Asset:Outlet:Component {{ {} }})".format( i, props_stm)) query.append("CREATE (ups)-[:HAS_COMPONENT]->(out{})".format(i)) query.append("CREATE (out{})-[:POWERED_BY]->(ups)".format(i)) session.run("\n".join(query))
def create_server(key, attr, server_variation=ServerVariations.Server): """Create a simulated server """ if not attr["power_consumption"]: raise KeyError("Server asset requires power_consumption attribute") if not attr["domain_name"]: raise KeyError("Must provide VM name (domain name)") # Validate server domain name try: conn = libvirt.open("qemu:///system") conn.lookupByName(attr["domain_name"]) except libvirt.libvirtError: raise KeyError("VM does not exist") finally: conn.close() with GRAPH_REF.get_session() as session: query = [] # cypher query attr["name"] = (attr["name"] if "name" in attr and attr["name"] else attr["domain_name"]) attr["type"] = server_variation.name.lower() attr["key"] = key s_attr = [ "domain_name", "power_consumption", "power_source", ] + CREATE_SHARED_ATTR props_stm = qh.get_props_stm(attr, supported_attr=s_attr) # create a server query.append("CREATE (server:Asset {{ {} }}) SET server :{}".format( props_stm, server_variation.name)) query.append("CREATE (server)-[:HAS_CPU]->(:CPU)") # set BMC-server specific attributes if type is bmc if server_variation == ServerVariations.ServerWithBMC: bmc_attr = {**IPMI_LAN_DEFAULTS, **attr} # merge set_stm = qh.get_set_stm(bmc_attr, node_name="server", supported_attr=IPMI_LAN_DEFAULTS.keys()) query.append("SET {}".format(set_stm)) session.run("\n".join(query)) if server_variation == ServerVariations.ServerWithBMC: # if preset is provided -> use the user-defined file f_loc = os.path.dirname(__file__) s_def_file = (lambda p, j: os.path.expanduser(attr[p]) if p in attr and attr[p] else os.path.join( f_loc, "presets/" + j)) sensor_file = s_def_file("sensor_def", "sensors.json") storage_def_file = s_def_file("storage_def", "storage.json") storage_state_file = s_def_file("storage_states", "storage_states.json") _add_sensors(key, sensor_file) _add_storage(key, storage_def_file, storage_state_file) if server_variation == ServerVariations.ServerWithBMC: with open(sensor_file) as preset_handler, GRAPH_REF.get_session( ) as session: data = json.load(preset_handler) for psu in data["psu"]: _add_psu(key, psu_index=psu["id"], attr=psu) else: # add PSUs to the model for i in range(attr["psu_num"]): psu_attr = { "power_consumption": attr["psu_power_consumption"], "power_source": attr["psu_power_source"], "variation": server_variation.name.lower(), "draw": attr["psu_load"][i] if attr["psu_load"] else 1, "id": i, } _add_psu(key, psu_index=i + 1, attr=psu_attr)
def _add_storage(asset_key, preset_file, storage_state_file): """Add storage to the server with asset_key""" with open(preset_file) as preset_h, open( storage_state_file) as state_h, GRAPH_REF.get_session() as session: query = [] query.append("MATCH (server:Asset {{ key: {} }})".format(asset_key)) storage_data = json.load(preset_h) state_data = json.load(state_h) props_stm = qh.get_props_stm( { **storage_data, **{ "stateConfig": json.dumps(state_data) } }, supported_attr=["operatingSystem", "CLIVersion", "stateConfig"], ) query.append( "CREATE (server)-[:SUPPORTS_STORCLI]->(storage:Storcli {{ {} }})". format(props_stm)) for idx, controller in enumerate(storage_data["controllers"]): s_attr = [ "controllerNum", "model", "serialNumber", "SASAddress", "PCIAddress", "mfgDate", "reworkDate", "memoryCorrectable_errors", "memoryUncorrectable_errors", "alarmState", "numDriveGroups", "bgiRate", "prRate", "rebuildRate", "ccRate", ] default_ctr_prop = { "memoryCorrectable_errors": 0, "memoryUncorrectable_errors": 0, "alarmState": "off", "controllerNum": idx, } props_stm = qh.get_props_stm({ **controller, **default_ctr_prop }, supported_attr=s_attr) ctrl_node = "ctrl" + str(idx) query.append( "CREATE (server)-[:HAS_CONTROLLER]->({}:Controller {{ {} }})". format(ctrl_node, props_stm)) # BBU or CacheVault if "BBU" in controller and controller["BBU"]: props_stm = qh.get_props_stm( controller["BBU"], supported_attr=[ "model", "serialNumber", "type", "replacementNeeded", "state", "designCapacity", ], ) query.append( "CREATE ({})-[:HAS_BBU]->(bbu:BBU {{ {} }})".format( ctrl_node, props_stm)) elif "CacheVault" in controller and controller["CacheVault"]: props_stm = qh.get_props_stm( { **controller["CacheVault"], **{ "temperature": 0, "replacement": "No", "writeThrough": True }, }, supported_attr=[ "model", "replacement", "state", "temperature", "mfgDate", "serialNumber", "writeThrough", ], ) query.append( "CREATE ({})-[:HAS_CACHEVAULT]->(cache:CacheVault {{ {} }})" .format(ctrl_node, props_stm)) # Add physical drives for pidx, phys_drive in enumerate(controller["PD"]): pd_node = "pd" + str(phys_drive["DID"]) # define supported attributes s_attr = [ "EID", "DID", "State", "DG", "Size", "Intf", "Med", "SED", "PI", "SeSz", "Model", "Sp", "Type", "PDC", "slotNum", "temperature", "mediaErrorCount", "otherErrorCount", "predictiveErrorCount", "rebuildTime", "timeStamp", "manufacturerId", "serialNumber", ] props_stm = qh.get_props_stm( { **phys_drive, **{ "slotNum": pidx, "mediaErrorCount": 0, "otherErrorCount": 0, "predictiveErrorCount": 0, "temperature": 0, "timeStamp": time.time(), }, }, supported_attr=s_attr, ) query.append( "CREATE ({})-[:HAS_PHYSICAL_DRIVE]->({}:PhysicalDrive {{ {} }})" .format(ctrl_node, pd_node, props_stm)) for vidx, virt_drive in enumerate(controller["VD"]): vd_node = "vd" + str(vidx) s_attr = [ "TYPE", "DG", "State", "Access", "Cac", "Cache", "Name", "Consist", "sCC", "Size", "vdNum", ] props_stm = qh.get_props_stm({ **virt_drive, **{ "vdNum": vidx } }, supported_attr=s_attr) query.append( "CREATE ({})-[:HAS_VIRTUAL_DRIVE]->({}:VirtualDrive {{ {} }})" .format(ctrl_node, vd_node, props_stm)) # connect PDs & VDs for pidx in virt_drive["DID"]: query.append( "CREATE ({})<-[:BELONGS_TO_VIRTUAL_SPACE]-(pd{})". format(vd_node, pidx)) # print("\n".join(query)) session.run("\n".join(query))
def _add_sensors(asset_key, preset_file): """Add sensors based on a preset file""" with open( preset_file) as preset_handler, GRAPH_REF.get_session() as session: query = [] query.append("MATCH (server:Asset {{ key: {} }})".format(asset_key)) data = json.load(preset_handler) for sensor_type, sensor_specs in data.items(): if sensor_type not in SUPPORTED_SENSORS: continue address_space_exists = ("addressSpace" in sensor_specs and sensor_specs["addressSpace"]) if address_space_exists: address_space = repr(sensor_specs["addressSpace"]) query.append( "CREATE (aSpace{}:AddressSpace {{ address: {} }})".format( sensor_specs["addressSpace"], address_space)) for idx, sensor in enumerate(sensor_specs["sensorDefinitions"]): sensor_node = "{}{}".format(sensor_type, idx) if "address" in sensor and sensor["address"]: addr = {"address": hex(int(sensor["address"], 16))} elif address_space_exists: addr = {"index": idx} else: raise KeyError( "Missing address for a sensor {}".format(sensor_type)) s_attr = [ "name", "defaultValue", "offValue", "group", "num", "lnr", "lcr", "lnc", "unc", "ucr", "unr", "address", "index", "type", "eventReadingType", ] sensor["name"] = sensor["name"].format( index=addr["index"] + 1 if "index" in addr else "") props = { **sensor["thresholds"], **sensor, **addr, **sensor_specs, **{ "type": sensor_type, "num": idx + 1 }, } props_stm = qh.get_props_stm(props, supported_attr=s_attr) query.append("CREATE (sensor{}:Sensor:{} {{ {} }})".format( sensor_node, sensor_type, props_stm)) if not ("address" in sensor) or not sensor["address"]: query.append( "CREATE (sensor{})-[:HAS_ADDRESS_SPACE]->(aSpace{})". format(sensor_node, sensor_specs["addressSpace"])) query.append( "CREATE (server)-[:HAS_SENSOR]->(sensor{})".format( sensor_node)) # print("\n".join(query)) session.run("\n".join(query))