def plays(cls):
"""Get plays (user-defined scripts) available for execution
Returns:
tuple: list of bash files as well as python scripts
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
play_path = GraphReference.get_play_path(session)
if not play_path:
return ([], [])
play_files = [
f
for f in os.listdir(play_path)
if os.path.isfile(os.path.join(play_path, f))
]
is_py_file = lambda f: os.path.splitext(f)[1] == ".py"
return (
[os.path.splitext(f)[0] for f in play_files if not is_py_file(f)],
[os.path.splitext(f)[0] for f in play_files if is_py_file(f)],
)
def _handle_status_request(self, details):
"""Get overall system status/details including hardware assets;
environment state & play details
"""
assets = IStateManager.get_system_status(flatten=False)
graph_ref = GraphReference()
with graph_ref.get_session() as session:
stage_layout = GraphReference.get_stage_layout(session)
# send system topology and assets' power-interconnections
self._write_data(
details["client"],
ServerToClientRequests.sys_layout,
{"assets": assets, "stageLayout": stage_layout},
)
self._write_data(
details["client"],
ServerToClientRequests.ambient_upd,
{"ambient": ISystemEnvironment.get_ambient(), "rising": False},
)
self._write_data(
details["client"],
ServerToClientRequests.play_list,
{"plays": list(itertools.chain(*IStateManager.plays()))},
)
self._write_data(
details["client"],
ServerToClientRequests.mains_upd,
{"mains": ISystemEnvironment.mains_status()},
)
def __init__(self, server_key, enable_thermal=False):
self._server_key = server_key
self._graph_ref = GraphReference()
self._sensor_file_locks = SensorFileLocks()
self._sensor_dir = os.path.join(get_temp_workplace_dir(),
str(server_key), "sensor_dir")
self._sensors = {}
with self._graph_ref.get_session() as session:
sensors = GraphReference.get_asset_sensors(session, server_key)
for sensor_info in sensors:
sensor = Sensor(
self._sensor_dir,
server_key,
sensor_info,
self._sensor_file_locks,
graph_ref=self._graph_ref,
)
self._sensors[sensor.name] = sensor
if enable_thermal:
self._load_thermal = True
if not os.path.isdir(self._sensor_dir):
os.mkdir(self._sensor_dir)
for s_name in self._sensors:
self._sensors[s_name].set_to_defaults()
def update_storage_temperature(self, old_ambient, new_ambient):
with self._graph_ref.get_session() as db_s:
hd_elements = GraphReference.get_all_hd_thermal_elements(
db_s, self.key)
for hd_e in hd_elements:
if "DID" in hd_e["component"]:
target_attr = "DID"
target_value = hd_e["component"]["DID"]
target_type = "PhysicalDrive"
else:
target_attr = "serialNumber"
target_value = '"{}"'.format(
hd_e["component"]["serialNumber"])
target_type = "CacheVault"
updated, new_temp = GraphReference.add_to_hd_component_temperature(
db_s,
target={
"server_key": self.key,
"controller": hd_e["controller"]["controllerNum"],
"attribute": target_attr,
"value": target_value,
"hd_type": target_type,
},
temp_change=new_ambient - old_ambient,
limit={
"lower": new_ambient,
"upper": None
},
)
def _init_thermal_impact(self):
"""Initialize thermal imact based on the saved inter-connections"""
with self._graph_ref.get_session() as session:
thermal_sensor_rel_details = GraphReference.get_affected_sensors(
session, self._server_key, self._s_name)
# for each target & for each set of relationships with the target
for target in thermal_sensor_rel_details["targets"]:
for rel in target["rel"]:
self._launch_thermal_sensor_thread(target["name"],
rel["event"])
thermal_storage_rel_details = GraphReference.get_affected_hd_elements(
session, self._server_key, self._s_name)
for target in thermal_storage_rel_details["targets"]:
if "DID" in target and target["DID"]:
hd_type = HDComponents.PhysicalDrive
hd_element = target["DID"]
else:
hd_type = HDComponents.CacheVault
hd_element = target["serialNumber"]
for rel in target["rel"]:
self._launch_thermal_storage_thread(
target["controller"]["controllerNum"],
hd_element,
hd_type,
rel["event"],
)
self._launch_thermal_cpu_thread()
def asset_exists(cls, key):
"""Check if asset with the key exists"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
asset_info = GraphReference.get_asset_and_components(session, key)
return asset_info is not None
def execute_play(cls, play_name):
"""Execute a specific play
Args:
play_name(str): playbook name
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
play_path = GraphReference.get_play_path(session)
if not play_path:
return
file_filter = (
lambda f: os.path.isfile(os.path.join(play_path, f))
and os.path.splitext(f)[0] == play_name
)
play_file = [f for f in os.listdir(play_path) if file_filter(f)][0]
subprocess.Popen(
os.path.join(play_path, play_file),
stderr=subprocess.DEVNULL,
close_fds=True,
)
def _handle_layout_request(self, details):
"""Save assets' positions/coordinates"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
GraphReference.save_layout(session,
details["payload"]["assets"],
stage=details["payload"]["stage"])
def initialize(force_snmp_init=False):
""" Initialize redis state using topology defined in the graph db """
graph_ref = GraphReference()
redis_store = redis.StrictRedis(host="localhost", port=6379)
with graph_ref.get_session() as session:
results = session.run(
"""
MATCH (asset:Asset) OPTIONAL MATCH (asset:Asset)-[:HAS_OID]->(oid)
return asset, collect(oid) as oids
"""
)
for record in results:
asset_type = record["asset"].get("type")
asset_key = str(record["asset"].get("key"))
init_from_snmprec = (
not redis_store.exists("{}-{}:state".format(asset_key, asset_type))
) or force_snmp_init
redis_store.set("{}-{}:state".format(asset_key, asset_type), 1)
formatted_key = asset_key.zfill(10)
temp_ordering_key = formatted_key + "-temp_oids_ordering"
graph_oids = {}
for oid in record["oids"]: # loop over oids that are defined in the graph db
graph_oids[oid.get("OID")] = {
"dtype": oid.get("dataType"),
"value": oid.get("defaultValue"),
}
# Set-up in the SNMPSim format
if "SNMPSim" in record["asset"].labels and record["oids"] and init_from_snmprec:
# Read a file containing static .snmprec data
static_oid_file = record["asset"].get("staticOidFile")
static_oid_path = os.path.join(
os.environ.get("SIMENGINE_STATIC_DATA"), static_oid_file
)
with open(static_oid_path, "r") as sfile_handler:
for line in sfile_handler:
oid, dtype, value = line.replace("\n", "").split("|")
if oid in graph_oids:
dtype = graph_oids[oid]["dtype"]
value = graph_oids[oid]["value"]
key_and_oid = format_as_redis_key(formatted_key, oid)
redis_store.lpush(temp_ordering_key, key_and_oid)
redis_store.set(key_and_oid, "{}|{}".format(dtype, value))
redis_store.sort(
temp_ordering_key, store=formatted_key + "-oids_ordering", alpha=True
)
redis_store.delete(temp_ordering_key)
redis_store.rpush(asset_key, formatted_key)
def get_ambient_props(cls) -> tuple:
"""Get runtime ambient properties (ambient behaviour description)
Returns:
thermal parameters for up/down events, randomizer ambient properties
returns None if System Environment hasn't been initialized yet
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
return GraphReference.get_ambient_props(session)
def get_voltage_props(cls) -> dict:
"""Get runtime voltage properties (ambient behaviour description)
Returns:
voltage fluctuation properties such as method being used (normal/gauss)
& properties associated with the random method
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
return GraphReference.get_voltage_props(session)
def _cpu_impact(self):
"""Keep updating *this sensor based on cpu load changes
This function waits for the thermal event switch and exits when the connection between this sensor & cpu load
is removed;
"""
# avoid circular imports with the server
from enginecore.state.api import IBMCServerStateManager
with self._graph_ref.get_session() as session:
asset_info = GraphReference.get_asset_and_components(
session, self._server_key)
server_sm = IBMCServerStateManager(asset_info)
cpu_impact_degrees_1 = 0
cpu_impact_degrees_2 = 0
while True:
self._s_thermal_event.wait()
rel_details = GraphReference.get_cpu_thermal_rel(
session, self._server_key, self.name)
# relationship was deleted
if not rel_details:
return
with self._s_file_locks.get_lock(self.name):
current_cpu_load = server_sm.cpu_load
# calculate cpu impact based on the model
cpu_impact_degrees_2 = self._calc_approx_value(
json.loads(rel_details["model"]), current_cpu_load)
new_calc_value = (int(self.sensor_value) +
cpu_impact_degrees_2 -
cpu_impact_degrees_1)
# meaning update is needed
if cpu_impact_degrees_1 != cpu_impact_degrees_2:
ambient = ISystemEnvironment.get_ambient()
self.sensor_value = (new_calc_value
if new_calc_value > ambient else
int(ambient))
logger.debug(
"Thermal impact of CPU load at (%s%%) updated: (%s°)->(%s°)",
current_cpu_load,
cpu_impact_degrees_1,
cpu_impact_degrees_2,
)
cpu_impact_degrees_1 = cpu_impact_degrees_2
time.sleep(5)
def set_ambient_props(cls, props):
"""Update runtime thermal properties of the room temperature
Args:
props: ambient behaviour specs such as "event" (upon what event: up/down),
"degrees" (how much rises/dropw), "rate" (seconds),
"pause_at" (should stop at this temperature)
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
GraphReference.set_ambient_props(session, props)
def _get_virtual_drives(self, controller_num):
"""Retrieve virtual drive data"""
drives = []
with self._graph_ref.get_session() as session:
vd_details = GraphReference.get_virtual_drive_details(
session, self._server_key, controller_num
)
cv_info = GraphReference.get_cachevault(
session, self._server_key, controller_num
)
# iterate over virtual drives
for i, v_drive in enumerate(vd_details):
vd_state = copy.deepcopy(
self._storcli_details["stateConfig"]["virtualDrive"]["Optl"]
)
# Add Virtual Drive output
v_drive["DG/VD"] = "0/" + str(i)
v_drive["Size"] = str(v_drive["Size"]) + " GB"
if cv_info["replacement"] == "Yes" and cv_info["writeThrough"]:
v_drive["Cache"] = "RWTD"
# Add physical drive output (do some formatting plus check pd states)
self._format_pd_for_output(v_drive["pd"])
self._check_vd_state(vd_state, v_drive["pd"])
v_drive["State"] = self._get_state_from_config(
"virtualDrive", vd_state, "Optl"
)
drives.append(
{
"physical_drives": self._format_as_table(
StorCLIEmulator.pd_header, v_drive["pd"]
),
"virtual_drives": self._format_as_table(
StorCLIEmulator.vd_header, [v_drive]
),
"virtual_drives_num": i,
}
)
return drives
def add_sensor_thermal_impact(self, target, event):
"""Set a target sensor that will be affected by the current source sensor values
Args:
target(str): Name of the target sensor
event(str): Source event causing the thermal impact to trigger
"""
if target in self._th_sensor_t and event in self._th_sensor_t[target]:
raise ValueError("Thread already exists")
with self._graph_ref.get_session() as session:
rel_details = GraphReference.get_sensor_thermal_rel(
session,
self._server_key,
relationship={
"source": self.name,
"target": {
"attribute": "name",
"value": '"{}"'.format(target)
},
"event": event,
},
)
if rel_details:
self._launch_thermal_sensor_thread(target,
rel_details["rel"]["event"])
def get_psu_sensor_names(self):
"""Find out BMC-specific psu keys (voltage, status etc.)
Returns:
dict: key value pairs of sensor type / sensor name for the psu
"""
with self._graph_ref.get_session() as db_s:
return GraphReference.get_psu_sensor_names(db_s, self.key, self._psu_number)
def set_cv_replacement(self, controller: int, repl_status: str,
wt_on_fail: bool):
"""Update Cachevault replacement status"""
with self._graph_ref.get_session() as session:
return GraphReference.set_cv_replacement(session, self.key,
controller, repl_status,
wt_on_fail)
def set_storage_randomizer_prop(self, proptype: StorageRandProps, slc: slice):
"""Update properties of randomized storage arguments"""
with self._graph_ref.get_session() as session:
return GraphReference.set_storage_randomizer_prop(
session, self.key, proptype.name, slc
)
def _parents_available(self):
"""Indicates whether a state action can be performed;
checks if parent nodes are up & running and all OIDs indicate 'on' status
Returns:
bool: True if parents are available
"""
asset_keys, oid_keys = GraphReference.get_parent_keys(
self._graph_ref.get_session(), self._asset_key
)
# if wall-powered, check the mains
if not asset_keys and not ISystemEnvironment.power_source_available():
return False
state_managers = list(map(self.get_state_manager_by_key, asset_keys))
min_volt_value = self.min_voltage_prop()
# check if power is present
enough_voltage = len(
list(filter(lambda sm: sm.output_voltage > min_volt_value, state_managers))
)
parent_assets_up = len(list(filter(lambda sm: sm.status, state_managers)))
oid_clause = (
lambda rvalue, rkey: rvalue.split(b"|")[1].decode()
== oid_keys[rkey]["switchOff"]
)
oids_on = self._check_parents(oid_keys.keys(), oid_clause)
return (parent_assets_up and enough_voltage and oids_on) or (not asset_keys)
def get_system_status(cls, flatten=True):
"""Get states of all system components
Args:
flatten(bool): If false, the returned assets in the dict
will have their child-components nested
Returns:
dict: Current information on assets including their states, load etc.
"""
graph_ref = GraphReference()
with graph_ref.get_session() as session:
# cache assets
assets = GraphReference.get_assets_and_connections(session, flatten)
return cls._get_assets_states(assets, flatten)
def get_state_manager_by_key(cls, key, supported_assets=None):
"""Infer asset manager from key"""
from enginecore.state.hardware.room import Asset
if not supported_assets:
supported_assets = Asset.get_supported_assets()
graph_ref = GraphReference()
with graph_ref.get_session() as session:
asset_info = GraphReference.get_asset_and_components(session, key)
sm_mro = supported_assets[asset_info["type"]].StateManagerCls.mro()
module = ".".join(__name__.split(".")[:-1]) # api module
return next(filter(lambda x: x.__module__.startswith(module),
sm_mro))(asset_info)
def __init__(self, asset_key, server_dir, socket_port):
self._graph_ref = GraphReference()
self._server_key = asset_key
self._serversocket = None
with self._graph_ref.get_session() as session:
self._storcli_details = GraphReference.get_storcli_details(
session, asset_key)
self._storcli_dir = os.path.join(server_dir, "storcli")
os.makedirs(self._storcli_dir)
dir_util.copy_tree(os.environ.get("SIMENGINE_STORCLI_TEMPL"),
self._storcli_dir)
self.start_server(asset_key, socket_port)
def get_oid_by_name(self, oid_name):
"""Get oid by oid name"""
with self._graph_ref.get_session() as db_s:
oid = ISnmpDeviceStateManager.ObjectIdentity(
*GraphReference.get_asset_oid_by_name(db_s, int(
self._asset_key), oid_name))
return oid
def _strcli_ctrl_phys_disks(self, controller_num):
"""Storcli physical drive details"""
pd_info_f = os.path.join(self._storcli_dir, "physical_disk_data")
pd_entry_f = os.path.join(self._storcli_dir, "physical_disk_entry")
pd_output = []
info_options = {
"header": self._strcli_header(controller_num),
"physical_drives": "",
}
with open(pd_info_f) as info_h, open(
pd_entry_f) as entry_h, self._graph_ref.get_session(
) as session:
drives = GraphReference.get_all_drives(session, self._server_key,
controller_num)
pd_template = entry_h.read()
pd_drives = sorted(drives["pd"], key=lambda k: k["slotNum"])
self._format_pd_for_output(pd_drives)
pd_drive_table = map(
lambda pd: {
"drive_path":
"/c{}/e{}/s{}".format(controller_num, pd["EID"], pd[
"slotNum"]),
"drive_table":
self._format_as_table(StorCLIEmulator.pd_header, [pd]),
"media_error_count":
pd["mediaErrorCount"],
"other_error_count":
pd["otherErrorCount"],
"predictive_failure_count":
pd["predictiveErrorCount"],
"drive_temp_c":
pd["temperature"],
"drive_temp_f": (pd["temperature"] * 9 / 5) + 32,
"drive_model":
pd["Model"],
"drive_size":
pd["Size"],
"drive_group":
pd["DG"],
"serial_number":
pd["serialNumber"],
"manufacturer_id":
pd["manufacturerId"],
},
pd_drives,
)
pd_output = map(Template(pd_template).substitute, pd_drive_table)
info_options["physical_drives"] = "\n".join(pd_output)
return Template(info_h.read()).substitute(info_options)
def get_oid_value_by_name(self, oid_name):
"""Get value under object id name"""
with self._graph_ref.get_session() as session:
oid, parent_key = GraphReference.get_component_oid_by_name(
session, self.key, oid_name)
if oid:
oid = "{}.{}".format(oid, str(self.key)[-1])
return int(self.get_oid_value(oid, key=parent_key))
return 0
def set_controller_prop(self, controller: int, properties: dict):
"""Update properties associated with a RAID controller
Args:
controller: id/number assigned to a controller
properties: controller props including "alarm", correctable & uncorrectable
errors as "mem_c_errors", "mem_uc_errors"
"""
with self._graph_ref.get_session() as session:
return GraphReference.set_controller_prop(session, self.key,
controller, properties)
def get_parent_assets(cls, asset_key):
"""Get parent asset keys (nodes that are powering the asset)
Args:
asset_key(int): child key
Returns:
list: parent asset keys
"""
with cls.graph_ref.get_session() as session:
parent_assets = GraphReference.get_parent_assets(
session, asset_key)
return [a["key"] for a in parent_assets]
def set_physical_drive_prop(self, controller: int, did: int, properties: dict):
"""Update properties of a physical drive belonging to a RAID array
Args:
controller: id of the controller physical drive is member of
did: DID - unique drive id
properties: props associated with physical drive
such as drive state ('state') or error counts
('media_error_count', 'other_error_count', 'predictive_error_count')
"""
with self._graph_ref.get_session() as session:
return GraphReference.set_physical_drive_prop(
session, self.key, controller, did, properties
)
class HardwareDataSource:
graph_ref = GraphReference()
@classmethod
def get_all_assets(cls):
return NotImplementedError()
@classmethod
def get_affected_assets(cls, asset_key):
return NotImplementedError()
@classmethod
def get_mains_powered_assets(cls):
return NotImplementedError()
def _strcli_ctrlcount(self):
"""Number of adapters per server """
template_f_path = os.path.join(self._storcli_dir, "adapter_count")
with open(template_f_path) as templ_h, self._graph_ref.get_session() as session:
options = {
"header": self._strcli_header(),
"ctrl_count": GraphReference.get_controller_count(
session, self._server_key
),
}
template = Template(templ_h.read())
return template.substitute(options)
