def plays(cls):
"""Get plays (user-defined scripts) available for execution
Returns:
tuple: list of bash files as well as python scripts
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
play_path = GraphReference.get_play_path(session)
if not play_path:
return ([], [])
play_files = [
f
for f in os.listdir(play_path)
if os.path.isfile(os.path.join(play_path, f))
]
is_py_file = lambda f: os.path.splitext(f)[1] == ".py"
return (
[os.path.splitext(f)[0] for f in play_files if not is_py_file(f)],
[os.path.splitext(f)[0] for f in play_files if is_py_file(f)],
)
def _handle_status_request(self, details):
"""Get overall system status/details including hardware assets;
environment state & play details
"""
assets = IStateManager.get_system_status(flatten=False)
graph_ref = GraphReference()
with graph_ref.get_session() as session:
stage_layout = GraphReference.get_stage_layout(session)
# send system topology and assets' power-interconnections
self._write_data(
details["client"],
ServerToClientRequests.sys_layout,
{"assets": assets, "stageLayout": stage_layout},
)
self._write_data(
details["client"],
ServerToClientRequests.ambient_upd,
{"ambient": ISystemEnvironment.get_ambient(), "rising": False},
)
self._write_data(
details["client"],
ServerToClientRequests.play_list,
{"plays": list(itertools.chain(*IStateManager.plays()))},
)
self._write_data(
details["client"],
ServerToClientRequests.mains_upd,
{"mains": ISystemEnvironment.mains_status()},
)
def asset_exists(cls, key):
"""Check if asset with the key exists"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
asset_info = GraphReference.get_asset_and_components(session, key)
return asset_info is not None
def execute_play(cls, play_name):
"""Execute a specific play
Args:
play_name(str): playbook name
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
play_path = GraphReference.get_play_path(session)
if not play_path:
return
file_filter = (
lambda f: os.path.isfile(os.path.join(play_path, f))
and os.path.splitext(f)[0] == play_name
)
play_file = [f for f in os.listdir(play_path) if file_filter(f)][0]
subprocess.Popen(
os.path.join(play_path, play_file),
stderr=subprocess.DEVNULL,
close_fds=True,
)
def _handle_layout_request(self, details):
"""Save assets' positions/coordinates"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
GraphReference.save_layout(session,
details["payload"]["assets"],
stage=details["payload"]["stage"])
def initialize(force_snmp_init=False):
""" Initialize redis state using topology defined in the graph db """
graph_ref = GraphReference()
redis_store = redis.StrictRedis(host="localhost", port=6379)
with graph_ref.get_session() as session:
results = session.run(
"""
MATCH (asset:Asset) OPTIONAL MATCH (asset:Asset)-[:HAS_OID]->(oid)
return asset, collect(oid) as oids
"""
)
for record in results:
asset_type = record["asset"].get("type")
asset_key = str(record["asset"].get("key"))
init_from_snmprec = (
not redis_store.exists("{}-{}:state".format(asset_key, asset_type))
) or force_snmp_init
redis_store.set("{}-{}:state".format(asset_key, asset_type), 1)
formatted_key = asset_key.zfill(10)
temp_ordering_key = formatted_key + "-temp_oids_ordering"
graph_oids = {}
for oid in record["oids"]: # loop over oids that are defined in the graph db
graph_oids[oid.get("OID")] = {
"dtype": oid.get("dataType"),
"value": oid.get("defaultValue"),
}
# Set-up in the SNMPSim format
if "SNMPSim" in record["asset"].labels and record["oids"] and init_from_snmprec:
# Read a file containing static .snmprec data
static_oid_file = record["asset"].get("staticOidFile")
static_oid_path = os.path.join(
os.environ.get("SIMENGINE_STATIC_DATA"), static_oid_file
)
with open(static_oid_path, "r") as sfile_handler:
for line in sfile_handler:
oid, dtype, value = line.replace("\n", "").split("|")
if oid in graph_oids:
dtype = graph_oids[oid]["dtype"]
value = graph_oids[oid]["value"]
key_and_oid = format_as_redis_key(formatted_key, oid)
redis_store.lpush(temp_ordering_key, key_and_oid)
redis_store.set(key_and_oid, "{}|{}".format(dtype, value))
redis_store.sort(
temp_ordering_key, store=formatted_key + "-oids_ordering", alpha=True
)
redis_store.delete(temp_ordering_key)
redis_store.rpush(asset_key, formatted_key)
def get_voltage_props(cls) -> dict:
"""Get runtime voltage properties (ambient behaviour description)
Returns:
voltage fluctuation properties such as method being used (normal/gauss)
& properties associated with the random method
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
return GraphReference.get_voltage_props(session)
def get_ambient_props(cls) -> tuple:
"""Get runtime ambient properties (ambient behaviour description)
Returns:
thermal parameters for up/down events, randomizer ambient properties
returns None if System Environment hasn't been initialized yet
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
return GraphReference.get_ambient_props(session)
def set_ambient_props(cls, props):
"""Update runtime thermal properties of the room temperature
Args:
props: ambient behaviour specs such as "event" (upon what event: up/down),
"degrees" (how much rises/dropw), "rate" (seconds),
"pause_at" (should stop at this temperature)
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
GraphReference.set_ambient_props(session, props)
def get_system_status(cls, flatten=True):
"""Get states of all system components
Args:
flatten(bool): If false, the returned assets in the dict
will have their child-components nested
Returns:
dict: Current information on assets including their states, load etc.
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
# cache assets
assets = GraphReference.get_assets_and_connections(session, flatten)
return cls._get_assets_states(assets, flatten)
def get_state_manager_by_key(cls, key, supported_assets=None):
"""Infer asset manager from key"""
from enginecore.state.hardware.room import Asset
if not supported_assets:
supported_assets = Asset.get_supported_assets()
graph_ref = GraphReference()
with graph_ref.get_session() as session:
asset_info = GraphReference.get_asset_and_components(session, key)
sm_mro = supported_assets[asset_info["type"]].StateManagerCls.mro()
module = ".".join(__name__.split(".")[:-1]) # api module
return next(filter(lambda x: x.__module__.startswith(module),
sm_mro))(asset_info)
class StorCLIEmulator:
"""This component emulates storcli behaviour
- runs a websocket server that listens to any incoming commands from a vm
"""
pd_header = [
"EID:Slt",
"DID",
"State",
"DG",
"Size",
"Intf",
"Med",
"SED",
"PI",
"SeSz",
"Model",
"Sp",
"Type",
]
vd_header = [
"DG/VD",
"TYPE",
"State",
"Access",
"Consist",
"Cache",
"Cac",
"sCC",
"Size",
"Name",
]
topology_header = [
"DG", # disk group idx
"Arr", # array idx
"Row",
"EID:Slot", # enclosure device ID
"DID",
"Type",
"State",
"BT", # background task
"Size",
"PDC", # pd cache
"PI", # protection info
"SED", # self encrypting drive
"DS3", # Dimmer Switch 3
"FSpace", # free space present
"TR", # transport ready
]
def __init__(self, asset_key, server_dir, socket_port):
self._graph_ref = GraphReference()
self._server_key = asset_key
self._serversocket = None
with self._graph_ref.get_session() as session:
self._storcli_details = GraphReference.get_storcli_details(
session, asset_key)
self._storcli_dir = os.path.join(server_dir, "storcli")
os.makedirs(self._storcli_dir)
dir_util.copy_tree(os.environ.get("SIMENGINE_STORCLI_TEMPL"),
self._storcli_dir)
self.start_server(asset_key, socket_port)
def start_server(self, asset_key, socket_port):
"""Launch storcli thread"""
self._stop_event = threading.Event()
self._socket_t = threading.Thread(
target=self._listen_cmds,
args=(socket_port, ),
name="storcli64:{}".format(asset_key),
)
self._socket_t.daemon = True
self._socket_t.start()
def stop_server(self):
"""Stop"""
self._stop_event.set()
self._serversocket.close()
# *** Responses to cli commands ***
def _strcli_header(self, ctrl_num=0, status="Success"):
"""Reusable header for storcli output
(this appears at the top of most CLI outputs)"""
with open(os.path.join(self._storcli_dir, "header")) as templ_h:
options = {
"cli_version":
self._storcli_details["CLIVersion"],
"op_sys":
self._storcli_details["operatingSystem"],
"status":
status,
"description":
"None",
"controller_line":
"Controller = {}\n".format(ctrl_num) if ctrl_num else "",
}
template = Template(templ_h.read())
return template.substitute(options)
def _strcli_ctrlcount(self):
"""Number of adapters per server"""
template_f_path = os.path.join(self._storcli_dir, "adapter_count")
with open(template_f_path) as templ_h, self._graph_ref.get_session(
) as session:
options = {
"header":
self._strcli_header(),
"ctrl_count":
GraphReference.get_controller_count(session, self._server_key),
}
template = Template(templ_h.read())
return template.substitute(options)
def _strcli_ctrl_perf_mode(self, controller_num):
"""Current performance mode (hardcoded)"""
with open(os.path.join(self._storcli_dir,
"performance_mode")) as templ_h:
options = {
"header": self._strcli_header(controller_num),
"mode_num": 0,
"mode_description": "tuned to provide Best IOPS",
}
template = Template(templ_h.read())
return template.substitute(options)
def _strcli_ctrl_alarm_state(self, controller_num):
"""Get controller alarm state"""
alarm_state_f_path = os.path.join(self._storcli_dir, "alarm_state")
with open(alarm_state_f_path) as templ_h, self._graph_ref.get_session(
) as session:
ctrl_info = GraphReference.get_controller_details(
session, self._server_key, controller_num)
options = {
"header": self._strcli_header(controller_num),
"alarm_state": ctrl_info["alarmState"],
}
template = Template(templ_h.read())
return template.substitute(options)
def _strcli_ctrl_bbu(self, controller_num):
"""Battery backup unit output for storcli"""
with open(os.path.join(self._storcli_dir, "bbu_data")) as templ_h:
options = {
"header": self._strcli_header(controller_num),
"ctrl_num": controller_num,
"status": "Failed",
"property": "-",
"err_msg": "use /cx/cv",
"err_code": 255,
}
template = Template(templ_h.read())
return template.substitute(options)
def _get_rate_prop(self, controller_num, rate_type):
"""Get controller rate property
(rate type matches rate template file and the rate template value)"""
rate_file = os.path.join(self._storcli_dir, rate_type)
with open(rate_file) as templ_h, self._graph_ref.get_session(
) as session:
ctrl_info = GraphReference.get_controller_details(
session, self._server_key, controller_num)
options = {}
options["header"] = self._strcli_header(controller_num)
options[rate_type] = ctrl_info[to_camelcase(rate_type)]
template = Template(templ_h.read())
return template.substitute(options)
def _check_vd_state(self, vd_state, physical_drives):
"""Determine status of the virtual drive based on the physical drives' states
Args:
vd_state(dict): virtual drive state properties such as error counts
& physical drive status
physical_drives(dict): physical drive details
"""
# Add physical drive output (do some formatting plus check pd states)
for p_drive in physical_drives:
vd_state["mediaErrorCount"] += p_drive["mediaErrorCount"]
vd_state["otherErrorCount"] += p_drive["otherErrorCount"]
vd_state["predictiveErrorCount"] += p_drive["predictiveErrorCount"]
pd_rebuilding = (time.time() -
p_drive["timeStamp"]) < p_drive["rebuildTime"]
if p_drive["State"] == "Offln" or pd_rebuilding:
vd_state["numPdOffline"] += 1
def _format_pd_for_output(self, physical_drives):
"""Update a table of physical drives so it is formatted as storcli output"""
for p_drive in physical_drives:
p_drive["EID:Slt"] = "{}:{}".format(p_drive["EID"],
p_drive["slotNum"])
p_drive["Size"] = str(p_drive["Size"]) + " GB"
def _strcli_ctrl_info(self, controller_num):
"""Return aggregated information for a particular controller (show all)"""
ctrl_info_f = os.path.join(self._storcli_dir, "controller_info")
ctrl_entry_f = os.path.join(self._storcli_dir, "controller_entry")
with open(ctrl_info_f) as info_h, open(
ctrl_entry_f) as entry_h, self._graph_ref.get_session(
) as session:
ctrl_info = GraphReference.get_controller_details(
session, self._server_key, controller_num)
topology_defaults = {
"DG": 0,
"Arr": "-",
"Row": "-",
"EID:Slot": "-",
"DID": "-",
"BT": "N",
"PDC": "dsbl",
"PI": "N",
"SED": "N",
"DS3": "none",
"FSpace": "N",
"TR": "N",
}
# templated keys
ctrl_info_templ_keys = [
"serial_number",
"model",
"serial_number",
"mfg_date",
"SAS_address",
"PCI_address",
"rework_date",
"memory_correctable_errors",
"memory_uncorrectable_errors",
"rebuild_rate",
"pr_rate",
"bgi_rate",
"cc_rate",
]
entry_options = {
"controller_num": controller_num,
"drive_groups_num": ctrl_info["numDriveGroups"],
"controller_date": "",
"system_date": "",
"status": "Optimal",
}
# copy over controller details
for key in ctrl_info_templ_keys:
entry_options[key] = ctrl_info[to_camelcase(key)]
# keep track of the issues associated with the controller
ctrl_state = copy.deepcopy(
self._storcli_details["stateConfig"]["controller"]["Optimal"])
ctrl_state["memoryCorrectableErrors"] = ctrl_info[
"memoryCorrectableErrors"]
ctrl_state["memoryUncorrectableErrors"] = ctrl_info[
"memoryUncorrectableErrors"]
drives = GraphReference.get_all_drives(session, self._server_key,
controller_num)
# Add physical drive output (do some formatting plus check pd states)
drives["pd"].sort(key=lambda k: k["slotNum"])
topology = []
# analyze and format virtual drive output
for i, v_drive in enumerate(drives["vd"]):
vd_state = copy.deepcopy(self._storcli_details["stateConfig"]
["virtualDrive"]["Optl"])
# Add Virtual Drive output
v_drive["DG/VD"] = str(v_drive["DG"]) + "/" + str(i)
v_drive["Size"] = str(v_drive["Size"]) + " GB"
# check pd states & determine virtual drive health status
self._check_vd_state(vd_state, drives["pd"])
v_drive["State"] = self._get_state_from_config(
"virtualDrive", vd_state, "Optl")
if v_drive["State"] != "Optl":
ctrl_state["vdDgd"] += 1
topology.extend([
{
**topology_defaults,
**{
"DG": v_drive["DG"],
"Type": v_drive["TYPE"],
"State": v_drive["State"],
"Size": v_drive["Size"],
},
},
{
**topology_defaults,
**{
"Arr": 0,
"DG": v_drive["DG"],
"Type": v_drive["TYPE"],
"State": v_drive["State"],
"Size": v_drive["Size"],
},
},
])
self._format_pd_for_output(v_drive["pd"])
for pdid, pd in enumerate(v_drive["pd"]):
topology.append({
**topology_defaults,
**{
"Arr": 0,
"DG": v_drive["DG"],
"Row": pdid,
"EID:Slot": pd["EID:Slt"],
"DID": pd["DID"],
"Type": "DRIVE",
"State": pd["State"],
"Size": pd["Size"],
"FSpace": "-",
},
})
self._format_pd_for_output(drives["pd"])
# determine overall controller status
entry_options["status"] = self._get_state_from_config(
"controller", ctrl_state, "Optimal")
# get cachevault details:
cv_info = GraphReference.get_cachevault(session, self._server_key,
controller_num)
cv_table = {
"Model": cv_info["model"],
"State": cv_info["state"],
"Temp": str(cv_info["temperature"]) + "C",
"Mode": "-",
"MfgDate": cv_info["mfgDate"],
}
return Template(info_h.read()).substitute({
"header":
self._strcli_header(controller_num),
"controller_entry":
Template(entry_h.read()).substitute(entry_options),
"num_virt_drives":
len(drives["vd"]),
"num_phys_drives":
len(drives["pd"]),
"topology":
self._format_as_table(StorCLIEmulator.topology_header,
topology),
"virtual_drives":
self._format_as_table(StorCLIEmulator.vd_header, drives["vd"]),
"physical_drives":
self._format_as_table(StorCLIEmulator.pd_header, drives["pd"]),
"cachevault":
self._format_as_table(cv_table.keys(), [cv_table]),
})
def _strcli_ctrl_cachevault(self, controller_num):
"""Cachevault output for storcli"""
cv_f = os.path.join(self._storcli_dir, "cachevault_data")
with open(os.path.join(
self._storcli_dir,
cv_f)) as templ_h, self._graph_ref.get_session() as session:
cv_info = GraphReference.get_cachevault(session, self._server_key,
controller_num)
cv_info["mfgDate"] = "/".join(
reversed(cv_info["mfgDate"].split(
"/"))) # dumb storcli (change date format)
options = {
**{
"header": self._strcli_header(controller_num)
},
**cv_info
}
template = Template(templ_h.read())
return template.substitute(options)
def _strcli_ctrl_phys_disks(self, controller_num):
"""Storcli physical drive details"""
pd_info_f = os.path.join(self._storcli_dir, "physical_disk_data")
pd_entry_f = os.path.join(self._storcli_dir, "physical_disk_entry")
pd_output = []
info_options = {
"header": self._strcli_header(controller_num),
"physical_drives": "",
}
with open(pd_info_f) as info_h, open(
pd_entry_f) as entry_h, self._graph_ref.get_session(
) as session:
drives = GraphReference.get_all_drives(session, self._server_key,
controller_num)
pd_template = entry_h.read()
pd_drives = sorted(drives["pd"], key=lambda k: k["slotNum"])
self._format_pd_for_output(pd_drives)
pd_drive_table = map(
lambda pd: {
"drive_path":
"/c{}/e{}/s{}".format(controller_num, pd["EID"], pd[
"slotNum"]),
"drive_table":
self._format_as_table(StorCLIEmulator.pd_header, [pd]),
"media_error_count":
pd["mediaErrorCount"],
"other_error_count":
pd["otherErrorCount"],
"predictive_failure_count":
pd["predictiveErrorCount"],
"drive_temp_c":
pd["temperature"],
"drive_temp_f": (pd["temperature"] * 9 / 5) + 32,
"drive_model":
pd["Model"],
"drive_size":
pd["Size"],
"drive_group":
pd["DG"],
"serial_number":
pd["serialNumber"],
"manufacturer_id":
pd["manufacturerId"],
},
pd_drives,
)
pd_output = map(Template(pd_template).substitute, pd_drive_table)
info_options["physical_drives"] = "\n".join(pd_output)
return Template(info_h.read()).substitute(info_options)
def _format_as_table(self, headers, table_options):
"""Formats data as storcli table
Args:
headers(list): table header
table_options(dict): table values
Returns:
str: storcli table populated with data
"""
value_rows = []
# store row with the max char count in a column
header_lengths = {key: len(str(key)) for key in headers}
left_align_cols = []
# calculate paddings for every column
for table_row in table_options:
for col_key in headers:
val_len = len(str(table_row[col_key]))
# whitespace padding, set to len of header if smaller than header
val_len = val_len if val_len >= len(col_key) else len(col_key)
if val_len > header_lengths[col_key]:
header_lengths[col_key] = val_len
if (not isinstance(table_row[col_key], (int, float, complex))
and not col_key == "Size"):
left_align_cols.append(col_key)
for table_row in table_options:
row_str = ""
for col_key in headers:
cell_value = table_row[col_key]
if col_key in left_align_cols:
cell_format = "{val:<{width}}"
else:
cell_format = "{val:>{width}}"
table_cell = cell_format.format(val=cell_value,
width=header_lengths[col_key])
row_str += table_cell + " "
value_rows.append(row_str)
header = " ".join([
"{val:<{width}}".format(val=key, width=header_lengths[key])
for key in header_lengths
])
divider = "-" * len(header) + "\n"
return str(divider + header + "\n" + divider + "\n".join(value_rows) +
"\n" + divider)
def _get_state_from_config(self, config_entry, state_map, optimal):
"""Configure state based on the configuration json"""
config = self._storcli_details["stateConfig"][config_entry]
state = optimal
for cnf_state in config:
if cnf_state == optimal:
continue
for hd_prop in state_map:
if (state_map[hd_prop] >= config[cnf_state][hd_prop]
and config[cnf_state][hd_prop] != -1):
state = cnf_state
return state
def _get_virtual_drives(self, controller_num):
"""Retrieve virtual drive data"""
drives = []
with self._graph_ref.get_session() as session:
vd_details = GraphReference.get_virtual_drive_details(
session, self._server_key, controller_num)
cv_info = GraphReference.get_cachevault(session, self._server_key,
controller_num)
# iterate over virtual drives
for i, v_drive in enumerate(vd_details):
vd_state = copy.deepcopy(self._storcli_details["stateConfig"]
["virtualDrive"]["Optl"])
# Add Virtual Drive output
v_drive["DG/VD"] = "0/" + str(i)
v_drive["Size"] = str(v_drive["Size"]) + " GB"
if cv_info["replacement"] == "Yes" and cv_info["writeThrough"]:
v_drive["Cache"] = "RWTD"
# Add physical drive output (do some formatting plus check pd states)
self._format_pd_for_output(v_drive["pd"])
self._check_vd_state(vd_state, v_drive["pd"])
v_drive["State"] = self._get_state_from_config(
"virtualDrive", vd_state, "Optl")
drives.append({
"physical_drives":
self._format_as_table(StorCLIEmulator.pd_header,
v_drive["pd"]),
"virtual_drives":
self._format_as_table(StorCLIEmulator.vd_header,
[v_drive]),
"virtual_drives_num":
i,
})
return drives
def _strcli_ctrl_virt_disk(self, controller_num):
"""Display virtual disk details"""
vd_file = os.path.join(self._storcli_dir, "virtual_drive_data")
with open(vd_file) as templ_h:
template = Template(templ_h.read())
# get virtual & physical drive details
drives = self._get_virtual_drives(controller_num)
vd_output = map(
lambda d: template.substitute({
**d,
**{
"controller": controller_num
}
}),
drives,
)
return self._strcli_header(controller_num) + "\n" + "\n".join(
vd_output)
def _listen_cmds(self, socket_port):
"""Start storcli websocket server
for a list of supported storcli64 commands, see
https://simengine.readthedocs.io/en/latest/Asset%20Management/#storage-simulation
"""
self._serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
1)
self._serversocket.bind(("", socket_port))
self._serversocket.listen(5)
# Continue to accept connections until the ws server stops
while not self._stop_event.is_set():
try:
conn, _ = self._serversocket.accept()
except OSError:
logger.warning("Could not initialize socket server!")
return
with conn:
while not self._stop_event.is_set():
scout_r, _, _ = select.select([conn], [], [])
if not scout_r:
logger.debug("Socket is not ready for reading")
continue
data = conn.recv(1024)
if not data:
logger.debug("Connection closed by remote end")
break
try:
received = json.loads(data)
except json.decoder.JSONDecodeError as parse_err:
logger.warning(data)
logger.warning("Invalid JSON: ")
logger.warning(parse_err)
argv = received["argv"]
logger.debug("Data received: %s", str(received))
reply = {"stdout": "", "stderr": "", "status": 0}
# Process non-default return cases
# (parse command request and return command output)
if len(argv) == 2:
if argv[1] == "--version":
reply["stdout"] = "Version 0.01"
elif len(argv) == 3:
if argv[1] == "show" and argv[2] == "ctrlcount":
reply["stdout"] = self._strcli_ctrlcount()
# Controller Commands
elif len(argv) == 4 and argv[1].startswith("/c"):
if argv[2] == "show" and argv[3] == "perfmode":
reply["stdout"] = self._strcli_ctrl_perf_mode(
argv[1][-1])
elif argv[2] == "show" and argv[3] == "bgirate":
reply["stdout"] = self._get_rate_prop(
argv[1][-1], "bgi_rate")
elif argv[2] == "show" and argv[3] == "ccrate":
reply["stdout"] = self._get_rate_prop(
argv[1][-1], "cc_rate")
elif argv[2] == "show" and argv[3] == "rebuildrate":
reply["stdout"] = self._get_rate_prop(
argv[1][-1], "rebuild_rate")
elif argv[2] == "show" and argv[3] == "prrate":
reply["stdout"] = self._get_rate_prop(
argv[1][-1], "pr_rate")
elif argv[2] == "show" and argv[3] == "alarm":
reply["stdout"] = self._strcli_ctrl_alarm_state(
argv[1][-1])
elif argv[2] == "show" and argv[3] == "all":
reply["stdout"] = self._strcli_ctrl_info(
argv[1][-1])
elif len(argv) == 5 and argv[1].startswith("/c"):
if argv[2] == "/bbu" and argv[3] == "show" and argv[
4] == "all":
reply["stdout"] = self._strcli_ctrl_bbu(
argv[1][-1])
elif (argv[2] == "/cv" and argv[3] == "show"
and argv[4] == "all"):
reply["stdout"] = self._strcli_ctrl_cachevault(
argv[1][-1])
elif (argv[2] == "/vall" and argv[3] == "show"
and argv[4] == "all"):
reply["stdout"] = self._strcli_ctrl_virt_disk(
argv[1][-1])
elif len(argv) == 6 and argv[1].startswith("/c"):
if (argv[2] == "/eall" and argv[3] == "/sall"
and argv[4] == "show" and argv[5] == "all"):
reply["stdout"] = self._strcli_ctrl_phys_disks(
argv[1][-1])
else:
reply = {
"stdout": "",
"stderr": "Usage: " + argv[0] + " --version",
"status": 1,
}
# Send the message
conn.sendall(bytes(json.dumps(reply) + "\n", "UTF-8"))
# Clean up connection when it gets closed by the remote end
conn.close()
# Final clean up when the ws server stops
self._serversocket.close()
class SensorRepository:
"""A sensor repository for a particular IPMI device"""
def __init__(self, server_key, enable_thermal=False):
self._server_key = server_key
self._graph_ref = GraphReference()
self._sensor_file_locks = SensorFileLocks()
self._sensor_dir = os.path.join(get_temp_workplace_dir(),
str(server_key), "sensor_dir")
self._sensors = {}
with self._graph_ref.get_session() as session:
sensors = GraphReference.get_asset_sensors(session, server_key)
for sensor_info in sensors:
sensor = Sensor(
self._sensor_dir,
server_key,
sensor_info,
self._sensor_file_locks,
graph_ref=self._graph_ref,
)
self._sensors[sensor.name] = sensor
if enable_thermal:
self._load_thermal = True
if not os.path.isdir(self._sensor_dir):
os.mkdir(self._sensor_dir)
for s_name in self._sensors:
self._sensors[s_name].set_to_defaults()
def __str__(self):
repo_str = []
repo_str.append("Sensor Repository for Server {}".format(
self._server_key))
repo_str.append(
" - files for sensor readings are located at '{}'".format(
self._sensor_dir))
return "\n\n".join(
repo_str +
list(map(lambda sn: str(self._sensors[sn]), self._sensors)))
def enable_thermal_impact(self):
"""Set thermal event switch """
list(
map(lambda sn: self._sensors[sn].enable_thermal_impact(),
self._sensors))
def disable_thermal_impact(self):
"""Clear thermal event switch"""
list(
map(lambda sn: self._sensors[sn].disable_thermal_impact(),
self._sensors))
def shut_down_sensors(self):
"""Set all sensors to offline"""
for s_name in self._sensors:
sensor = self._sensors[s_name]
if sensor.group != SensorGroups.temperature:
sensor.set_to_off()
self.disable_thermal_impact()
def power_up_sensors(self):
"""Set all sensors to online"""
for s_name in self._sensors:
sensor = self._sensors[s_name]
if sensor.group != SensorGroups.temperature:
sensor.set_to_defaults()
self.enable_thermal_impact()
def get_sensor_by_name(self, name):
"""Get a specific sensor by name"""
return self._sensors[name]
def get_sensors_by_group(self, group):
"""Get sensors by group name (temperature, fan etc) """
sensors_in_group = filter(lambda k: self._sensors[k].group == group,
self._sensors)
return list(map(lambda k: self._sensors[k], sensors_in_group))
def adjust_thermal_sensors(self, old_ambient, new_ambient):
"""Indicate an ambient update"""
for s_name in self._sensors:
sensor = self._sensors[s_name]
if sensor.group == SensorGroups.temperature:
with self._sensor_file_locks.get_lock(sensor.name):
old_sensor_value = int(sensor.sensor_value)
new_sensor_value = (old_sensor_value - old_ambient +
new_ambient
if old_sensor_value else new_ambient)
logger.debug(
"Sensor:[%s] updated from %s° to %s° due to ambient changes (%s°)->(%s°)",
sensor.name,
old_sensor_value,
new_sensor_value,
old_ambient,
new_ambient,
)
sensor.sensor_value = int(new_sensor_value)
if self._load_thermal is True:
self._load_thermal = False
for s_name in self._sensors:
self._sensors[s_name].start_thermal_impact()
@property
def sensor_dir(self):
"""Get temp IPMI state dir"""
return self._sensor_dir
@property
def sensors(self):
"""Get all sensors in a sensor repo"""
return self._sensors
@property
def server_key(self):
"""Get key of the server repo belongs to"""
return self._server_key
def stop(self):
"""Closes all the open connections"""
self._graph_ref.close()
def set_voltage_props(cls, props):
"""Update runtime voltage properties of the mains power voltage"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
GraphReference.set_voltage_props(session, props)
def set_play_path(cls, path):
"""Update play folder containing scripts"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
GraphReference.set_play_path(session, path)
class IStateManager:
"""Base class for all the state managers """
redis_store = None
class PowerStateReason(Enum):
"""Describes reason behind asset power state"""
ac_restored = 1
ac_lost = 2
turned_on = 3
turned_off = 4
signal_on = 5
signal_off = 6
def __init__(self, asset_info):
self._graph_ref = GraphReference()
self._asset_key = asset_info["key"]
self._asset_info = asset_info
def close_connection(self):
"""Close bolt driver connections"""
self._graph_ref.close()
@property
def key(self) -> int:
"""Asset Key """
return self._asset_key
@property
def redis_key(self) -> str:
"""Asset key in redis format as '{key}-{type}' """
return "{}-{}".format(str(self.key), self.asset_type)
@property
def asset_type(self) -> str:
"""Asset Type """
return self._asset_info["type"]
@property
def power_on_ac_restored(self) -> bool:
"""Returns true if asset is configured to power up when AC
is restored (voltage is above lower threshold)"""
return (
self._asset_info["powerOnAc"] if "powerOnAc" in self._asset_info else True
)
@property
def power_usage(self):
"""Normal power usage in AMPS when powered up"""
if self.input_voltage and "powerConsumption" in self._asset_info:
return self._asset_info["powerConsumption"] / self.input_voltage
return 0.0
@property
def draw_percentage(self) -> float:
"""How much power the asset draws"""
return self._asset_info["draw"] if "draw" in self._asset_info else 1
@property
def power_consumption(self):
"""How much power this device consumes (wattage)"""
return (
self._asset_info["powerConsumption"]
if "powerConsumption" in self._asset_info
else 0
)
@property
def asset_info(self) -> dict:
"""Get information associated with the asset"""
return self._asset_info
@property
def load(self):
"""Get current load stored in redis (in AMPs)"""
return float(IStateManager.get_store().get(self.redis_key + ":load"))
@property
def wattage(self):
"""Asset wattage (assumes power-source to be 120v)"""
return self.load * self.input_voltage
def min_voltage_prop(self):
"""Get minimum voltage required and the poweroff timeout associated with it"""
if not "minVoltage" in self._asset_info:
return 0
return self._asset_info["minVoltage"]
@property
def status(self):
"""Operational State
Returns:
int: 1 if on, 0 if off
"""
return int(IStateManager.get_store().get(self.redis_key + ":state"))
@property
def input_voltage(self):
"""Asset input voltage in Volts"""
in_volt = IStateManager.get_store().get(self.redis_key + ":in-voltage")
return float(in_volt) if in_volt else 0.0
@property
def output_voltage(self):
"""Output voltage for the device"""
return self.input_voltage * self.status
@property
def agent(self):
"""Agent instance details (if supported)
Returns:
tuple: process id and status of the process (if it's running)
"""
pid = IStateManager.get_store().get(self.redis_key + ":agent")
return (
(int(pid), os.path.exists("/proc/" + pid.decode("utf-8"))) if pid else None
)
@record
@Randomizer.randomize_method()
def shut_down(self):
"""Implements state logic for graceful power-off event,
sleeps for the pre-configured time
Returns:
int: Asset's status after power-off operation
"""
if self.status:
self._sleep_shutdown()
self._set_state_off()
return 0
return self.status
@record
@Randomizer.randomize_method()
def power_off(self):
"""Implements state logic for abrupt power loss
Returns:
int: Asset's status after power-off operation
"""
if self.status:
self._set_state_off()
return 0
return self.status
@record
@Randomizer.randomize_method()
def power_up(self):
"""Implements state logic for power up;
sleeps for the pre-configured time & resets boot time
Returns:
int: Asset's status after power-on operation
"""
if self._parents_available() and not self.status:
self._sleep_powerup()
# update machine start time & turn on
self._reset_boot_time()
self._set_state_on()
return 1
return self.status
def _update_input_voltage(self, voltage: float):
"""Set input voltage"""
voltage = max(voltage, 0)
IStateManager.get_store().set(self.redis_key + ":in-voltage", voltage)
def _update_load(self, load: float):
"""Update power load for the asset"""
load = max(load, 0.0)
IStateManager.get_store().set(self.redis_key + ":load", load)
def _sleep_delay(self, delay_type):
"""Sleep for n number of ms determined by the delay_type"""
if delay_type in self._asset_info:
time.sleep(self._asset_info[delay_type] / 1000.0) # ms to sec
def _sleep_shutdown(self):
"""Hardware-specific shutdown delay"""
self._sleep_delay("offDelay")
def _sleep_powerup(self):
"""Hardware-specific powerup delay"""
self._sleep_delay("onDelay")
def _set_redis_asset_state(self, state):
"""Update redis value of the asset power status"""
IStateManager.get_store().set(self.redis_key + ":state", state)
def _set_state_on(self):
"""Set state to online"""
self._publish_power(self.status, 1)
def _set_state_off(self):
"""Set state to offline"""
self._publish_power(self.status, 0)
def _publish_power(self, old_state, new_state):
"""Notify daemon of power updates"""
IStateManager.get_store().publish(
RedisChannels.state_update_channel,
json.dumps(
{"key": self.key, "old_state": old_state, "new_state": new_state}
),
)
def _reset_boot_time(self):
"""Reset device start time (this redis key/value is used to calculate uptime)
(see snmppub.lua)
"""
IStateManager.get_store().set(
str(self._asset_key) + ":start_time", int(time.time())
)
def _check_parents(
self, keys, parent_down, msg="Cannot perform the action: [{}] parent is off"
):
"""Check that redis values pass certain condition
Args:
keys (list): Redis keys (formatted as required)
parent_down (callable): lambda clause
msg (str, optional): Error message to be printed
Returns:
bool: True if parent keys are missing or all parents
were verified with parent_down clause
"""
if not keys:
return True
parent_values = IStateManager.get_store().mget(keys)
pdown_msg = []
for rkey, rvalue in zip(keys, parent_values):
if parent_down(rvalue, rkey):
pdown_msg.append(msg.format(rkey))
if len(pdown_msg) == len(keys):
print("\n".join(pdown_msg))
return False
return True
def _parents_available(self):
"""Indicates whether a state action can be performed;
checks if parent nodes are up & running and all OIDs indicate 'on' status
Returns:
bool: True if parents are available
"""
asset_keys, oid_keys = GraphReference.get_parent_keys(
self._graph_ref.get_session(), self._asset_key
)
# if wall-powered, check the mains
if not asset_keys and not ISystemEnvironment.power_source_available():
return False
state_managers = list(map(self.get_state_manager_by_key, asset_keys))
min_volt_value = self.min_voltage_prop()
# check if power is present
enough_voltage = len(
list(filter(lambda sm: sm.output_voltage > min_volt_value, state_managers))
)
parent_assets_up = len(list(filter(lambda sm: sm.status, state_managers)))
oid_clause = (
lambda rvalue, rkey: rvalue.split(b"|")[1].decode()
== oid_keys[rkey]["switchOff"]
)
oids_on = self._check_parents(oid_keys.keys(), oid_clause)
return (parent_assets_up and enough_voltage and oids_on) or (not asset_keys)
def __str__(self):
return (
"Asset[{0.asset_type}][{0.key}] \n"
" - Status: {0.status} \n"
" - Load: {0.load}A\n"
" - Power Consumption: {0.power_consumption}W \n"
" - Input Voltage: {0.input_voltage}V\n"
" - Output Voltage: {0.output_voltage}V\n"
).format(self)
@classmethod
def get_store(cls):
"""Get redis db handler """
if not cls.redis_store:
cls.redis_store = redis.StrictRedis(host="localhost", port=6379)
return cls.redis_store
@classmethod
def _get_assets_states(cls, assets, flatten=True):
"""Query redis store and find states for each asset
Args:
flatten(bool): If false, the returned assets in the dict
will have their child-components nested
Returns:
dict: Current information on assets including their states, load etc.
"""
asset_keys = assets.keys()
if not asset_keys:
return None
asset_values = cls.get_store().mget(
list(map(lambda k: "{}-{}:state".format(k, assets[k]["type"]), asset_keys))
)
for rkey, rvalue in zip(assets, asset_values):
asset_state = cls.get_state_manager_by_key(rkey)
assets[rkey]["status"] = int(rvalue)
assets[rkey]["load"] = asset_state.load
if assets[rkey]["type"] == "ups":
assets[rkey]["battery"] = asset_state.battery_level
if not flatten and "children" in assets[rkey]:
# call recursively on children
assets[rkey]["children"] = cls._get_assets_states(
assets[rkey]["children"]
)
return assets
@classmethod
def get_system_status(cls, flatten=True):
"""Get states of all system components
Args:
flatten(bool): If false, the returned assets in the dict
will have their child-components nested
Returns:
dict: Current information on assets including their states, load etc.
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
# cache assets
assets = GraphReference.get_assets_and_connections(session, flatten)
return cls._get_assets_states(assets, flatten)
@classmethod
@lru_cache(maxsize=32)
def get_state_manager_by_key(cls, key, supported_assets=None):
"""Infer asset manager from key"""
from enginecore.state.hardware.room import Asset
if not supported_assets:
supported_assets = Asset.get_supported_assets()
graph_ref = GraphReference()
with graph_ref.get_session() as session:
asset_info = GraphReference.get_asset_and_components(session, key)
sm_mro = supported_assets[asset_info["type"]].StateManagerCls.mro()
module = ".".join(__name__.split(".")[:-1]) # api module
return next(filter(lambda x: x.__module__.startswith(module), sm_mro))(
asset_info
)
@classmethod
def asset_exists(cls, key):
"""Check if asset with the key exists"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
asset_info = GraphReference.get_asset_and_components(session, key)
return asset_info is not None
@classmethod
def set_play_path(cls, path):
"""Update play folder containing scripts"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
GraphReference.set_play_path(session, path)
@classmethod
def plays(cls):
"""Get plays (user-defined scripts) available for execution
Returns:
tuple: list of bash files as well as python scripts
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
play_path = GraphReference.get_play_path(session)
if not play_path:
return ([], [])
play_files = [
f
for f in os.listdir(play_path)
if os.path.isfile(os.path.join(play_path, f))
]
is_py_file = lambda f: os.path.splitext(f)[1] == ".py"
return (
[os.path.splitext(f)[0] for f in play_files if not is_py_file(f)],
[os.path.splitext(f)[0] for f in play_files if is_py_file(f)],
)
@classmethod
def execute_play(cls, play_name):
"""Execute a specific play
Args:
play_name(str): playbook name
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
play_path = GraphReference.get_play_path(session)
if not play_path:
return
file_filter = (
lambda f: os.path.isfile(os.path.join(play_path, f))
and os.path.splitext(f)[0] == play_name
)
play_file = [f for f in os.listdir(play_path) if file_filter(f)][0]
subprocess.Popen(
os.path.join(play_path, play_file),
stderr=subprocess.DEVNULL,
close_fds=True,
)
def get_thermal_cpu_details(cls, asset_key):
"""Query existing cpu->sensor relationship"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
return GraphReference.get_thermal_cpu_details(session, asset_key)
def get_sensor_definitions(cls, asset_key):
"""Get sensor definitions """
graph_ref = GraphReference()
with graph_ref.get_session() as session:
return GraphReference.get_asset_sensors(session, asset_key)
