def plays(cls): """Get plays (user-defined scripts) available for execution Returns: tuple: list of bash files as well as python scripts """ graph_ref = GraphReference() with graph_ref.get_session() as session: play_path = GraphReference.get_play_path(session) if not play_path: return ([], []) play_files = [ f for f in os.listdir(play_path) if os.path.isfile(os.path.join(play_path, f)) ] is_py_file = lambda f: os.path.splitext(f)[1] == ".py" return ( [os.path.splitext(f)[0] for f in play_files if not is_py_file(f)], [os.path.splitext(f)[0] for f in play_files if is_py_file(f)], )
def _handle_status_request(self, details): """Get overall system status/details including hardware assets; environment state & play details """ assets = IStateManager.get_system_status(flatten=False) graph_ref = GraphReference() with graph_ref.get_session() as session: stage_layout = GraphReference.get_stage_layout(session) # send system topology and assets' power-interconnections self._write_data( details["client"], ServerToClientRequests.sys_layout, {"assets": assets, "stageLayout": stage_layout}, ) self._write_data( details["client"], ServerToClientRequests.ambient_upd, {"ambient": ISystemEnvironment.get_ambient(), "rising": False}, ) self._write_data( details["client"], ServerToClientRequests.play_list, {"plays": list(itertools.chain(*IStateManager.plays()))}, ) self._write_data( details["client"], ServerToClientRequests.mains_upd, {"mains": ISystemEnvironment.mains_status()}, )
def asset_exists(cls, key): """Check if asset with the key exists""" graph_ref = GraphReference() with graph_ref.get_session() as session: asset_info = GraphReference.get_asset_and_components(session, key) return asset_info is not None
def execute_play(cls, play_name): """Execute a specific play Args: play_name(str): playbook name """ graph_ref = GraphReference() with graph_ref.get_session() as session: play_path = GraphReference.get_play_path(session) if not play_path: return file_filter = ( lambda f: os.path.isfile(os.path.join(play_path, f)) and os.path.splitext(f)[0] == play_name ) play_file = [f for f in os.listdir(play_path) if file_filter(f)][0] subprocess.Popen( os.path.join(play_path, play_file), stderr=subprocess.DEVNULL, close_fds=True, )
def _handle_layout_request(self, details): """Save assets' positions/coordinates""" graph_ref = GraphReference() with graph_ref.get_session() as session: GraphReference.save_layout(session, details["payload"]["assets"], stage=details["payload"]["stage"])
def initialize(force_snmp_init=False): """ Initialize redis state using topology defined in the graph db """ graph_ref = GraphReference() redis_store = redis.StrictRedis(host="localhost", port=6379) with graph_ref.get_session() as session: results = session.run( """ MATCH (asset:Asset) OPTIONAL MATCH (asset:Asset)-[:HAS_OID]->(oid) return asset, collect(oid) as oids """ ) for record in results: asset_type = record["asset"].get("type") asset_key = str(record["asset"].get("key")) init_from_snmprec = ( not redis_store.exists("{}-{}:state".format(asset_key, asset_type)) ) or force_snmp_init redis_store.set("{}-{}:state".format(asset_key, asset_type), 1) formatted_key = asset_key.zfill(10) temp_ordering_key = formatted_key + "-temp_oids_ordering" graph_oids = {} for oid in record["oids"]: # loop over oids that are defined in the graph db graph_oids[oid.get("OID")] = { "dtype": oid.get("dataType"), "value": oid.get("defaultValue"), } # Set-up in the SNMPSim format if "SNMPSim" in record["asset"].labels and record["oids"] and init_from_snmprec: # Read a file containing static .snmprec data static_oid_file = record["asset"].get("staticOidFile") static_oid_path = os.path.join( os.environ.get("SIMENGINE_STATIC_DATA"), static_oid_file ) with open(static_oid_path, "r") as sfile_handler: for line in sfile_handler: oid, dtype, value = line.replace("\n", "").split("|") if oid in graph_oids: dtype = graph_oids[oid]["dtype"] value = graph_oids[oid]["value"] key_and_oid = format_as_redis_key(formatted_key, oid) redis_store.lpush(temp_ordering_key, key_and_oid) redis_store.set(key_and_oid, "{}|{}".format(dtype, value)) redis_store.sort( temp_ordering_key, store=formatted_key + "-oids_ordering", alpha=True ) redis_store.delete(temp_ordering_key) redis_store.rpush(asset_key, formatted_key)
def get_voltage_props(cls) -> dict: """Get runtime voltage properties (ambient behaviour description) Returns: voltage fluctuation properties such as method being used (normal/gauss) & properties associated with the random method """ graph_ref = GraphReference() with graph_ref.get_session() as session: return GraphReference.get_voltage_props(session)
def get_ambient_props(cls) -> tuple: """Get runtime ambient properties (ambient behaviour description) Returns: thermal parameters for up/down events, randomizer ambient properties returns None if System Environment hasn't been initialized yet """ graph_ref = GraphReference() with graph_ref.get_session() as session: return GraphReference.get_ambient_props(session)
def set_ambient_props(cls, props): """Update runtime thermal properties of the room temperature Args: props: ambient behaviour specs such as "event" (upon what event: up/down), "degrees" (how much rises/dropw), "rate" (seconds), "pause_at" (should stop at this temperature) """ graph_ref = GraphReference() with graph_ref.get_session() as session: GraphReference.set_ambient_props(session, props)
def get_system_status(cls, flatten=True): """Get states of all system components Args: flatten(bool): If false, the returned assets in the dict will have their child-components nested Returns: dict: Current information on assets including their states, load etc. """ graph_ref = GraphReference() with graph_ref.get_session() as session: # cache assets assets = GraphReference.get_assets_and_connections(session, flatten) return cls._get_assets_states(assets, flatten)
def get_state_manager_by_key(cls, key, supported_assets=None): """Infer asset manager from key""" from enginecore.state.hardware.room import Asset if not supported_assets: supported_assets = Asset.get_supported_assets() graph_ref = GraphReference() with graph_ref.get_session() as session: asset_info = GraphReference.get_asset_and_components(session, key) sm_mro = supported_assets[asset_info["type"]].StateManagerCls.mro() module = ".".join(__name__.split(".")[:-1]) # api module return next(filter(lambda x: x.__module__.startswith(module), sm_mro))(asset_info)
class StorCLIEmulator: """This component emulates storcli behaviour - runs a websocket server that listens to any incoming commands from a vm """ pd_header = [ "EID:Slt", "DID", "State", "DG", "Size", "Intf", "Med", "SED", "PI", "SeSz", "Model", "Sp", "Type", ] vd_header = [ "DG/VD", "TYPE", "State", "Access", "Consist", "Cache", "Cac", "sCC", "Size", "Name", ] topology_header = [ "DG", # disk group idx "Arr", # array idx "Row", "EID:Slot", # enclosure device ID "DID", "Type", "State", "BT", # background task "Size", "PDC", # pd cache "PI", # protection info "SED", # self encrypting drive "DS3", # Dimmer Switch 3 "FSpace", # free space present "TR", # transport ready ] def __init__(self, asset_key, server_dir, socket_port): self._graph_ref = GraphReference() self._server_key = asset_key self._serversocket = None with self._graph_ref.get_session() as session: self._storcli_details = GraphReference.get_storcli_details( session, asset_key) self._storcli_dir = os.path.join(server_dir, "storcli") os.makedirs(self._storcli_dir) dir_util.copy_tree(os.environ.get("SIMENGINE_STORCLI_TEMPL"), self._storcli_dir) self.start_server(asset_key, socket_port) def start_server(self, asset_key, socket_port): """Launch storcli thread""" self._stop_event = threading.Event() self._socket_t = threading.Thread( target=self._listen_cmds, args=(socket_port, ), name="storcli64:{}".format(asset_key), ) self._socket_t.daemon = True self._socket_t.start() def stop_server(self): """Stop""" self._stop_event.set() self._serversocket.close() # *** Responses to cli commands *** def _strcli_header(self, ctrl_num=0, status="Success"): """Reusable header for storcli output (this appears at the top of most CLI outputs)""" with open(os.path.join(self._storcli_dir, "header")) as templ_h: options = { "cli_version": self._storcli_details["CLIVersion"], "op_sys": self._storcli_details["operatingSystem"], "status": status, "description": "None", "controller_line": "Controller = {}\n".format(ctrl_num) if ctrl_num else "", } template = Template(templ_h.read()) return template.substitute(options) def _strcli_ctrlcount(self): """Number of adapters per server""" template_f_path = os.path.join(self._storcli_dir, "adapter_count") with open(template_f_path) as templ_h, self._graph_ref.get_session( ) as session: options = { "header": self._strcli_header(), "ctrl_count": GraphReference.get_controller_count(session, self._server_key), } template = Template(templ_h.read()) return template.substitute(options) def _strcli_ctrl_perf_mode(self, controller_num): """Current performance mode (hardcoded)""" with open(os.path.join(self._storcli_dir, "performance_mode")) as templ_h: options = { "header": self._strcli_header(controller_num), "mode_num": 0, "mode_description": "tuned to provide Best IOPS", } template = Template(templ_h.read()) return template.substitute(options) def _strcli_ctrl_alarm_state(self, controller_num): """Get controller alarm state""" alarm_state_f_path = os.path.join(self._storcli_dir, "alarm_state") with open(alarm_state_f_path) as templ_h, self._graph_ref.get_session( ) as session: ctrl_info = GraphReference.get_controller_details( session, self._server_key, controller_num) options = { "header": self._strcli_header(controller_num), "alarm_state": ctrl_info["alarmState"], } template = Template(templ_h.read()) return template.substitute(options) def _strcli_ctrl_bbu(self, controller_num): """Battery backup unit output for storcli""" with open(os.path.join(self._storcli_dir, "bbu_data")) as templ_h: options = { "header": self._strcli_header(controller_num), "ctrl_num": controller_num, "status": "Failed", "property": "-", "err_msg": "use /cx/cv", "err_code": 255, } template = Template(templ_h.read()) return template.substitute(options) def _get_rate_prop(self, controller_num, rate_type): """Get controller rate property (rate type matches rate template file and the rate template value)""" rate_file = os.path.join(self._storcli_dir, rate_type) with open(rate_file) as templ_h, self._graph_ref.get_session( ) as session: ctrl_info = GraphReference.get_controller_details( session, self._server_key, controller_num) options = {} options["header"] = self._strcli_header(controller_num) options[rate_type] = ctrl_info[to_camelcase(rate_type)] template = Template(templ_h.read()) return template.substitute(options) def _check_vd_state(self, vd_state, physical_drives): """Determine status of the virtual drive based on the physical drives' states Args: vd_state(dict): virtual drive state properties such as error counts & physical drive status physical_drives(dict): physical drive details """ # Add physical drive output (do some formatting plus check pd states) for p_drive in physical_drives: vd_state["mediaErrorCount"] += p_drive["mediaErrorCount"] vd_state["otherErrorCount"] += p_drive["otherErrorCount"] vd_state["predictiveErrorCount"] += p_drive["predictiveErrorCount"] pd_rebuilding = (time.time() - p_drive["timeStamp"]) < p_drive["rebuildTime"] if p_drive["State"] == "Offln" or pd_rebuilding: vd_state["numPdOffline"] += 1 def _format_pd_for_output(self, physical_drives): """Update a table of physical drives so it is formatted as storcli output""" for p_drive in physical_drives: p_drive["EID:Slt"] = "{}:{}".format(p_drive["EID"], p_drive["slotNum"]) p_drive["Size"] = str(p_drive["Size"]) + " GB" def _strcli_ctrl_info(self, controller_num): """Return aggregated information for a particular controller (show all)""" ctrl_info_f = os.path.join(self._storcli_dir, "controller_info") ctrl_entry_f = os.path.join(self._storcli_dir, "controller_entry") with open(ctrl_info_f) as info_h, open( ctrl_entry_f) as entry_h, self._graph_ref.get_session( ) as session: ctrl_info = GraphReference.get_controller_details( session, self._server_key, controller_num) topology_defaults = { "DG": 0, "Arr": "-", "Row": "-", "EID:Slot": "-", "DID": "-", "BT": "N", "PDC": "dsbl", "PI": "N", "SED": "N", "DS3": "none", "FSpace": "N", "TR": "N", } # templated keys ctrl_info_templ_keys = [ "serial_number", "model", "serial_number", "mfg_date", "SAS_address", "PCI_address", "rework_date", "memory_correctable_errors", "memory_uncorrectable_errors", "rebuild_rate", "pr_rate", "bgi_rate", "cc_rate", ] entry_options = { "controller_num": controller_num, "drive_groups_num": ctrl_info["numDriveGroups"], "controller_date": "", "system_date": "", "status": "Optimal", } # copy over controller details for key in ctrl_info_templ_keys: entry_options[key] = ctrl_info[to_camelcase(key)] # keep track of the issues associated with the controller ctrl_state = copy.deepcopy( self._storcli_details["stateConfig"]["controller"]["Optimal"]) ctrl_state["memoryCorrectableErrors"] = ctrl_info[ "memoryCorrectableErrors"] ctrl_state["memoryUncorrectableErrors"] = ctrl_info[ "memoryUncorrectableErrors"] drives = GraphReference.get_all_drives(session, self._server_key, controller_num) # Add physical drive output (do some formatting plus check pd states) drives["pd"].sort(key=lambda k: k["slotNum"]) topology = [] # analyze and format virtual drive output for i, v_drive in enumerate(drives["vd"]): vd_state = copy.deepcopy(self._storcli_details["stateConfig"] ["virtualDrive"]["Optl"]) # Add Virtual Drive output v_drive["DG/VD"] = str(v_drive["DG"]) + "/" + str(i) v_drive["Size"] = str(v_drive["Size"]) + " GB" # check pd states & determine virtual drive health status self._check_vd_state(vd_state, drives["pd"]) v_drive["State"] = self._get_state_from_config( "virtualDrive", vd_state, "Optl") if v_drive["State"] != "Optl": ctrl_state["vdDgd"] += 1 topology.extend([ { **topology_defaults, **{ "DG": v_drive["DG"], "Type": v_drive["TYPE"], "State": v_drive["State"], "Size": v_drive["Size"], }, }, { **topology_defaults, **{ "Arr": 0, "DG": v_drive["DG"], "Type": v_drive["TYPE"], "State": v_drive["State"], "Size": v_drive["Size"], }, }, ]) self._format_pd_for_output(v_drive["pd"]) for pdid, pd in enumerate(v_drive["pd"]): topology.append({ **topology_defaults, **{ "Arr": 0, "DG": v_drive["DG"], "Row": pdid, "EID:Slot": pd["EID:Slt"], "DID": pd["DID"], "Type": "DRIVE", "State": pd["State"], "Size": pd["Size"], "FSpace": "-", }, }) self._format_pd_for_output(drives["pd"]) # determine overall controller status entry_options["status"] = self._get_state_from_config( "controller", ctrl_state, "Optimal") # get cachevault details: cv_info = GraphReference.get_cachevault(session, self._server_key, controller_num) cv_table = { "Model": cv_info["model"], "State": cv_info["state"], "Temp": str(cv_info["temperature"]) + "C", "Mode": "-", "MfgDate": cv_info["mfgDate"], } return Template(info_h.read()).substitute({ "header": self._strcli_header(controller_num), "controller_entry": Template(entry_h.read()).substitute(entry_options), "num_virt_drives": len(drives["vd"]), "num_phys_drives": len(drives["pd"]), "topology": self._format_as_table(StorCLIEmulator.topology_header, topology), "virtual_drives": self._format_as_table(StorCLIEmulator.vd_header, drives["vd"]), "physical_drives": self._format_as_table(StorCLIEmulator.pd_header, drives["pd"]), "cachevault": self._format_as_table(cv_table.keys(), [cv_table]), }) def _strcli_ctrl_cachevault(self, controller_num): """Cachevault output for storcli""" cv_f = os.path.join(self._storcli_dir, "cachevault_data") with open(os.path.join( self._storcli_dir, cv_f)) as templ_h, self._graph_ref.get_session() as session: cv_info = GraphReference.get_cachevault(session, self._server_key, controller_num) cv_info["mfgDate"] = "/".join( reversed(cv_info["mfgDate"].split( "/"))) # dumb storcli (change date format) options = { **{ "header": self._strcli_header(controller_num) }, **cv_info } template = Template(templ_h.read()) return template.substitute(options) def _strcli_ctrl_phys_disks(self, controller_num): """Storcli physical drive details""" pd_info_f = os.path.join(self._storcli_dir, "physical_disk_data") pd_entry_f = os.path.join(self._storcli_dir, "physical_disk_entry") pd_output = [] info_options = { "header": self._strcli_header(controller_num), "physical_drives": "", } with open(pd_info_f) as info_h, open( pd_entry_f) as entry_h, self._graph_ref.get_session( ) as session: drives = GraphReference.get_all_drives(session, self._server_key, controller_num) pd_template = entry_h.read() pd_drives = sorted(drives["pd"], key=lambda k: k["slotNum"]) self._format_pd_for_output(pd_drives) pd_drive_table = map( lambda pd: { "drive_path": "/c{}/e{}/s{}".format(controller_num, pd["EID"], pd[ "slotNum"]), "drive_table": self._format_as_table(StorCLIEmulator.pd_header, [pd]), "media_error_count": pd["mediaErrorCount"], "other_error_count": pd["otherErrorCount"], "predictive_failure_count": pd["predictiveErrorCount"], "drive_temp_c": pd["temperature"], "drive_temp_f": (pd["temperature"] * 9 / 5) + 32, "drive_model": pd["Model"], "drive_size": pd["Size"], "drive_group": pd["DG"], "serial_number": pd["serialNumber"], "manufacturer_id": pd["manufacturerId"], }, pd_drives, ) pd_output = map(Template(pd_template).substitute, pd_drive_table) info_options["physical_drives"] = "\n".join(pd_output) return Template(info_h.read()).substitute(info_options) def _format_as_table(self, headers, table_options): """Formats data as storcli table Args: headers(list): table header table_options(dict): table values Returns: str: storcli table populated with data """ value_rows = [] # store row with the max char count in a column header_lengths = {key: len(str(key)) for key in headers} left_align_cols = [] # calculate paddings for every column for table_row in table_options: for col_key in headers: val_len = len(str(table_row[col_key])) # whitespace padding, set to len of header if smaller than header val_len = val_len if val_len >= len(col_key) else len(col_key) if val_len > header_lengths[col_key]: header_lengths[col_key] = val_len if (not isinstance(table_row[col_key], (int, float, complex)) and not col_key == "Size"): left_align_cols.append(col_key) for table_row in table_options: row_str = "" for col_key in headers: cell_value = table_row[col_key] if col_key in left_align_cols: cell_format = "{val:<{width}}" else: cell_format = "{val:>{width}}" table_cell = cell_format.format(val=cell_value, width=header_lengths[col_key]) row_str += table_cell + " " value_rows.append(row_str) header = " ".join([ "{val:<{width}}".format(val=key, width=header_lengths[key]) for key in header_lengths ]) divider = "-" * len(header) + "\n" return str(divider + header + "\n" + divider + "\n".join(value_rows) + "\n" + divider) def _get_state_from_config(self, config_entry, state_map, optimal): """Configure state based on the configuration json""" config = self._storcli_details["stateConfig"][config_entry] state = optimal for cnf_state in config: if cnf_state == optimal: continue for hd_prop in state_map: if (state_map[hd_prop] >= config[cnf_state][hd_prop] and config[cnf_state][hd_prop] != -1): state = cnf_state return state def _get_virtual_drives(self, controller_num): """Retrieve virtual drive data""" drives = [] with self._graph_ref.get_session() as session: vd_details = GraphReference.get_virtual_drive_details( session, self._server_key, controller_num) cv_info = GraphReference.get_cachevault(session, self._server_key, controller_num) # iterate over virtual drives for i, v_drive in enumerate(vd_details): vd_state = copy.deepcopy(self._storcli_details["stateConfig"] ["virtualDrive"]["Optl"]) # Add Virtual Drive output v_drive["DG/VD"] = "0/" + str(i) v_drive["Size"] = str(v_drive["Size"]) + " GB" if cv_info["replacement"] == "Yes" and cv_info["writeThrough"]: v_drive["Cache"] = "RWTD" # Add physical drive output (do some formatting plus check pd states) self._format_pd_for_output(v_drive["pd"]) self._check_vd_state(vd_state, v_drive["pd"]) v_drive["State"] = self._get_state_from_config( "virtualDrive", vd_state, "Optl") drives.append({ "physical_drives": self._format_as_table(StorCLIEmulator.pd_header, v_drive["pd"]), "virtual_drives": self._format_as_table(StorCLIEmulator.vd_header, [v_drive]), "virtual_drives_num": i, }) return drives def _strcli_ctrl_virt_disk(self, controller_num): """Display virtual disk details""" vd_file = os.path.join(self._storcli_dir, "virtual_drive_data") with open(vd_file) as templ_h: template = Template(templ_h.read()) # get virtual & physical drive details drives = self._get_virtual_drives(controller_num) vd_output = map( lambda d: template.substitute({ **d, **{ "controller": controller_num } }), drives, ) return self._strcli_header(controller_num) + "\n" + "\n".join( vd_output) def _listen_cmds(self, socket_port): """Start storcli websocket server for a list of supported storcli64 commands, see https://simengine.readthedocs.io/en/latest/Asset%20Management/#storage-simulation """ self._serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self._serversocket.bind(("", socket_port)) self._serversocket.listen(5) # Continue to accept connections until the ws server stops while not self._stop_event.is_set(): try: conn, _ = self._serversocket.accept() except OSError: logger.warning("Could not initialize socket server!") return with conn: while not self._stop_event.is_set(): scout_r, _, _ = select.select([conn], [], []) if not scout_r: logger.debug("Socket is not ready for reading") continue data = conn.recv(1024) if not data: logger.debug("Connection closed by remote end") break try: received = json.loads(data) except json.decoder.JSONDecodeError as parse_err: logger.warning(data) logger.warning("Invalid JSON: ") logger.warning(parse_err) argv = received["argv"] logger.debug("Data received: %s", str(received)) reply = {"stdout": "", "stderr": "", "status": 0} # Process non-default return cases # (parse command request and return command output) if len(argv) == 2: if argv[1] == "--version": reply["stdout"] = "Version 0.01" elif len(argv) == 3: if argv[1] == "show" and argv[2] == "ctrlcount": reply["stdout"] = self._strcli_ctrlcount() # Controller Commands elif len(argv) == 4 and argv[1].startswith("/c"): if argv[2] == "show" and argv[3] == "perfmode": reply["stdout"] = self._strcli_ctrl_perf_mode( argv[1][-1]) elif argv[2] == "show" and argv[3] == "bgirate": reply["stdout"] = self._get_rate_prop( argv[1][-1], "bgi_rate") elif argv[2] == "show" and argv[3] == "ccrate": reply["stdout"] = self._get_rate_prop( argv[1][-1], "cc_rate") elif argv[2] == "show" and argv[3] == "rebuildrate": reply["stdout"] = self._get_rate_prop( argv[1][-1], "rebuild_rate") elif argv[2] == "show" and argv[3] == "prrate": reply["stdout"] = self._get_rate_prop( argv[1][-1], "pr_rate") elif argv[2] == "show" and argv[3] == "alarm": reply["stdout"] = self._strcli_ctrl_alarm_state( argv[1][-1]) elif argv[2] == "show" and argv[3] == "all": reply["stdout"] = self._strcli_ctrl_info( argv[1][-1]) elif len(argv) == 5 and argv[1].startswith("/c"): if argv[2] == "/bbu" and argv[3] == "show" and argv[ 4] == "all": reply["stdout"] = self._strcli_ctrl_bbu( argv[1][-1]) elif (argv[2] == "/cv" and argv[3] == "show" and argv[4] == "all"): reply["stdout"] = self._strcli_ctrl_cachevault( argv[1][-1]) elif (argv[2] == "/vall" and argv[3] == "show" and argv[4] == "all"): reply["stdout"] = self._strcli_ctrl_virt_disk( argv[1][-1]) elif len(argv) == 6 and argv[1].startswith("/c"): if (argv[2] == "/eall" and argv[3] == "/sall" and argv[4] == "show" and argv[5] == "all"): reply["stdout"] = self._strcli_ctrl_phys_disks( argv[1][-1]) else: reply = { "stdout": "", "stderr": "Usage: " + argv[0] + " --version", "status": 1, } # Send the message conn.sendall(bytes(json.dumps(reply) + "\n", "UTF-8")) # Clean up connection when it gets closed by the remote end conn.close() # Final clean up when the ws server stops self._serversocket.close()
class SensorRepository: """A sensor repository for a particular IPMI device""" def __init__(self, server_key, enable_thermal=False): self._server_key = server_key self._graph_ref = GraphReference() self._sensor_file_locks = SensorFileLocks() self._sensor_dir = os.path.join(get_temp_workplace_dir(), str(server_key), "sensor_dir") self._sensors = {} with self._graph_ref.get_session() as session: sensors = GraphReference.get_asset_sensors(session, server_key) for sensor_info in sensors: sensor = Sensor( self._sensor_dir, server_key, sensor_info, self._sensor_file_locks, graph_ref=self._graph_ref, ) self._sensors[sensor.name] = sensor if enable_thermal: self._load_thermal = True if not os.path.isdir(self._sensor_dir): os.mkdir(self._sensor_dir) for s_name in self._sensors: self._sensors[s_name].set_to_defaults() def __str__(self): repo_str = [] repo_str.append("Sensor Repository for Server {}".format( self._server_key)) repo_str.append( " - files for sensor readings are located at '{}'".format( self._sensor_dir)) return "\n\n".join( repo_str + list(map(lambda sn: str(self._sensors[sn]), self._sensors))) def enable_thermal_impact(self): """Set thermal event switch """ list( map(lambda sn: self._sensors[sn].enable_thermal_impact(), self._sensors)) def disable_thermal_impact(self): """Clear thermal event switch""" list( map(lambda sn: self._sensors[sn].disable_thermal_impact(), self._sensors)) def shut_down_sensors(self): """Set all sensors to offline""" for s_name in self._sensors: sensor = self._sensors[s_name] if sensor.group != SensorGroups.temperature: sensor.set_to_off() self.disable_thermal_impact() def power_up_sensors(self): """Set all sensors to online""" for s_name in self._sensors: sensor = self._sensors[s_name] if sensor.group != SensorGroups.temperature: sensor.set_to_defaults() self.enable_thermal_impact() def get_sensor_by_name(self, name): """Get a specific sensor by name""" return self._sensors[name] def get_sensors_by_group(self, group): """Get sensors by group name (temperature, fan etc) """ sensors_in_group = filter(lambda k: self._sensors[k].group == group, self._sensors) return list(map(lambda k: self._sensors[k], sensors_in_group)) def adjust_thermal_sensors(self, old_ambient, new_ambient): """Indicate an ambient update""" for s_name in self._sensors: sensor = self._sensors[s_name] if sensor.group == SensorGroups.temperature: with self._sensor_file_locks.get_lock(sensor.name): old_sensor_value = int(sensor.sensor_value) new_sensor_value = (old_sensor_value - old_ambient + new_ambient if old_sensor_value else new_ambient) logger.debug( "Sensor:[%s] updated from %s° to %s° due to ambient changes (%s°)->(%s°)", sensor.name, old_sensor_value, new_sensor_value, old_ambient, new_ambient, ) sensor.sensor_value = int(new_sensor_value) if self._load_thermal is True: self._load_thermal = False for s_name in self._sensors: self._sensors[s_name].start_thermal_impact() @property def sensor_dir(self): """Get temp IPMI state dir""" return self._sensor_dir @property def sensors(self): """Get all sensors in a sensor repo""" return self._sensors @property def server_key(self): """Get key of the server repo belongs to""" return self._server_key def stop(self): """Closes all the open connections""" self._graph_ref.close()
def set_voltage_props(cls, props): """Update runtime voltage properties of the mains power voltage""" graph_ref = GraphReference() with graph_ref.get_session() as session: GraphReference.set_voltage_props(session, props)
def set_play_path(cls, path): """Update play folder containing scripts""" graph_ref = GraphReference() with graph_ref.get_session() as session: GraphReference.set_play_path(session, path)
class IStateManager: """Base class for all the state managers """ redis_store = None class PowerStateReason(Enum): """Describes reason behind asset power state""" ac_restored = 1 ac_lost = 2 turned_on = 3 turned_off = 4 signal_on = 5 signal_off = 6 def __init__(self, asset_info): self._graph_ref = GraphReference() self._asset_key = asset_info["key"] self._asset_info = asset_info def close_connection(self): """Close bolt driver connections""" self._graph_ref.close() @property def key(self) -> int: """Asset Key """ return self._asset_key @property def redis_key(self) -> str: """Asset key in redis format as '{key}-{type}' """ return "{}-{}".format(str(self.key), self.asset_type) @property def asset_type(self) -> str: """Asset Type """ return self._asset_info["type"] @property def power_on_ac_restored(self) -> bool: """Returns true if asset is configured to power up when AC is restored (voltage is above lower threshold)""" return ( self._asset_info["powerOnAc"] if "powerOnAc" in self._asset_info else True ) @property def power_usage(self): """Normal power usage in AMPS when powered up""" if self.input_voltage and "powerConsumption" in self._asset_info: return self._asset_info["powerConsumption"] / self.input_voltage return 0.0 @property def draw_percentage(self) -> float: """How much power the asset draws""" return self._asset_info["draw"] if "draw" in self._asset_info else 1 @property def power_consumption(self): """How much power this device consumes (wattage)""" return ( self._asset_info["powerConsumption"] if "powerConsumption" in self._asset_info else 0 ) @property def asset_info(self) -> dict: """Get information associated with the asset""" return self._asset_info @property def load(self): """Get current load stored in redis (in AMPs)""" return float(IStateManager.get_store().get(self.redis_key + ":load")) @property def wattage(self): """Asset wattage (assumes power-source to be 120v)""" return self.load * self.input_voltage def min_voltage_prop(self): """Get minimum voltage required and the poweroff timeout associated with it""" if not "minVoltage" in self._asset_info: return 0 return self._asset_info["minVoltage"] @property def status(self): """Operational State Returns: int: 1 if on, 0 if off """ return int(IStateManager.get_store().get(self.redis_key + ":state")) @property def input_voltage(self): """Asset input voltage in Volts""" in_volt = IStateManager.get_store().get(self.redis_key + ":in-voltage") return float(in_volt) if in_volt else 0.0 @property def output_voltage(self): """Output voltage for the device""" return self.input_voltage * self.status @property def agent(self): """Agent instance details (if supported) Returns: tuple: process id and status of the process (if it's running) """ pid = IStateManager.get_store().get(self.redis_key + ":agent") return ( (int(pid), os.path.exists("/proc/" + pid.decode("utf-8"))) if pid else None ) @record @Randomizer.randomize_method() def shut_down(self): """Implements state logic for graceful power-off event, sleeps for the pre-configured time Returns: int: Asset's status after power-off operation """ if self.status: self._sleep_shutdown() self._set_state_off() return 0 return self.status @record @Randomizer.randomize_method() def power_off(self): """Implements state logic for abrupt power loss Returns: int: Asset's status after power-off operation """ if self.status: self._set_state_off() return 0 return self.status @record @Randomizer.randomize_method() def power_up(self): """Implements state logic for power up; sleeps for the pre-configured time & resets boot time Returns: int: Asset's status after power-on operation """ if self._parents_available() and not self.status: self._sleep_powerup() # update machine start time & turn on self._reset_boot_time() self._set_state_on() return 1 return self.status def _update_input_voltage(self, voltage: float): """Set input voltage""" voltage = max(voltage, 0) IStateManager.get_store().set(self.redis_key + ":in-voltage", voltage) def _update_load(self, load: float): """Update power load for the asset""" load = max(load, 0.0) IStateManager.get_store().set(self.redis_key + ":load", load) def _sleep_delay(self, delay_type): """Sleep for n number of ms determined by the delay_type""" if delay_type in self._asset_info: time.sleep(self._asset_info[delay_type] / 1000.0) # ms to sec def _sleep_shutdown(self): """Hardware-specific shutdown delay""" self._sleep_delay("offDelay") def _sleep_powerup(self): """Hardware-specific powerup delay""" self._sleep_delay("onDelay") def _set_redis_asset_state(self, state): """Update redis value of the asset power status""" IStateManager.get_store().set(self.redis_key + ":state", state) def _set_state_on(self): """Set state to online""" self._publish_power(self.status, 1) def _set_state_off(self): """Set state to offline""" self._publish_power(self.status, 0) def _publish_power(self, old_state, new_state): """Notify daemon of power updates""" IStateManager.get_store().publish( RedisChannels.state_update_channel, json.dumps( {"key": self.key, "old_state": old_state, "new_state": new_state} ), ) def _reset_boot_time(self): """Reset device start time (this redis key/value is used to calculate uptime) (see snmppub.lua) """ IStateManager.get_store().set( str(self._asset_key) + ":start_time", int(time.time()) ) def _check_parents( self, keys, parent_down, msg="Cannot perform the action: [{}] parent is off" ): """Check that redis values pass certain condition Args: keys (list): Redis keys (formatted as required) parent_down (callable): lambda clause msg (str, optional): Error message to be printed Returns: bool: True if parent keys are missing or all parents were verified with parent_down clause """ if not keys: return True parent_values = IStateManager.get_store().mget(keys) pdown_msg = [] for rkey, rvalue in zip(keys, parent_values): if parent_down(rvalue, rkey): pdown_msg.append(msg.format(rkey)) if len(pdown_msg) == len(keys): print("\n".join(pdown_msg)) return False return True def _parents_available(self): """Indicates whether a state action can be performed; checks if parent nodes are up & running and all OIDs indicate 'on' status Returns: bool: True if parents are available """ asset_keys, oid_keys = GraphReference.get_parent_keys( self._graph_ref.get_session(), self._asset_key ) # if wall-powered, check the mains if not asset_keys and not ISystemEnvironment.power_source_available(): return False state_managers = list(map(self.get_state_manager_by_key, asset_keys)) min_volt_value = self.min_voltage_prop() # check if power is present enough_voltage = len( list(filter(lambda sm: sm.output_voltage > min_volt_value, state_managers)) ) parent_assets_up = len(list(filter(lambda sm: sm.status, state_managers))) oid_clause = ( lambda rvalue, rkey: rvalue.split(b"|")[1].decode() == oid_keys[rkey]["switchOff"] ) oids_on = self._check_parents(oid_keys.keys(), oid_clause) return (parent_assets_up and enough_voltage and oids_on) or (not asset_keys) def __str__(self): return ( "Asset[{0.asset_type}][{0.key}] \n" " - Status: {0.status} \n" " - Load: {0.load}A\n" " - Power Consumption: {0.power_consumption}W \n" " - Input Voltage: {0.input_voltage}V\n" " - Output Voltage: {0.output_voltage}V\n" ).format(self) @classmethod def get_store(cls): """Get redis db handler """ if not cls.redis_store: cls.redis_store = redis.StrictRedis(host="localhost", port=6379) return cls.redis_store @classmethod def _get_assets_states(cls, assets, flatten=True): """Query redis store and find states for each asset Args: flatten(bool): If false, the returned assets in the dict will have their child-components nested Returns: dict: Current information on assets including their states, load etc. """ asset_keys = assets.keys() if not asset_keys: return None asset_values = cls.get_store().mget( list(map(lambda k: "{}-{}:state".format(k, assets[k]["type"]), asset_keys)) ) for rkey, rvalue in zip(assets, asset_values): asset_state = cls.get_state_manager_by_key(rkey) assets[rkey]["status"] = int(rvalue) assets[rkey]["load"] = asset_state.load if assets[rkey]["type"] == "ups": assets[rkey]["battery"] = asset_state.battery_level if not flatten and "children" in assets[rkey]: # call recursively on children assets[rkey]["children"] = cls._get_assets_states( assets[rkey]["children"] ) return assets @classmethod def get_system_status(cls, flatten=True): """Get states of all system components Args: flatten(bool): If false, the returned assets in the dict will have their child-components nested Returns: dict: Current information on assets including their states, load etc. """ graph_ref = GraphReference() with graph_ref.get_session() as session: # cache assets assets = GraphReference.get_assets_and_connections(session, flatten) return cls._get_assets_states(assets, flatten) @classmethod @lru_cache(maxsize=32) def get_state_manager_by_key(cls, key, supported_assets=None): """Infer asset manager from key""" from enginecore.state.hardware.room import Asset if not supported_assets: supported_assets = Asset.get_supported_assets() graph_ref = GraphReference() with graph_ref.get_session() as session: asset_info = GraphReference.get_asset_and_components(session, key) sm_mro = supported_assets[asset_info["type"]].StateManagerCls.mro() module = ".".join(__name__.split(".")[:-1]) # api module return next(filter(lambda x: x.__module__.startswith(module), sm_mro))( asset_info ) @classmethod def asset_exists(cls, key): """Check if asset with the key exists""" graph_ref = GraphReference() with graph_ref.get_session() as session: asset_info = GraphReference.get_asset_and_components(session, key) return asset_info is not None @classmethod def set_play_path(cls, path): """Update play folder containing scripts""" graph_ref = GraphReference() with graph_ref.get_session() as session: GraphReference.set_play_path(session, path) @classmethod def plays(cls): """Get plays (user-defined scripts) available for execution Returns: tuple: list of bash files as well as python scripts """ graph_ref = GraphReference() with graph_ref.get_session() as session: play_path = GraphReference.get_play_path(session) if not play_path: return ([], []) play_files = [ f for f in os.listdir(play_path) if os.path.isfile(os.path.join(play_path, f)) ] is_py_file = lambda f: os.path.splitext(f)[1] == ".py" return ( [os.path.splitext(f)[0] for f in play_files if not is_py_file(f)], [os.path.splitext(f)[0] for f in play_files if is_py_file(f)], ) @classmethod def execute_play(cls, play_name): """Execute a specific play Args: play_name(str): playbook name """ graph_ref = GraphReference() with graph_ref.get_session() as session: play_path = GraphReference.get_play_path(session) if not play_path: return file_filter = ( lambda f: os.path.isfile(os.path.join(play_path, f)) and os.path.splitext(f)[0] == play_name ) play_file = [f for f in os.listdir(play_path) if file_filter(f)][0] subprocess.Popen( os.path.join(play_path, play_file), stderr=subprocess.DEVNULL, close_fds=True, )
def get_thermal_cpu_details(cls, asset_key): """Query existing cpu->sensor relationship""" graph_ref = GraphReference() with graph_ref.get_session() as session: return GraphReference.get_thermal_cpu_details(session, asset_key)
def get_sensor_definitions(cls, asset_key): """Get sensor definitions """ graph_ref = GraphReference() with graph_ref.get_session() as session: return GraphReference.get_asset_sensors(session, asset_key)