def handle_term(signum, frame):
global wdfile
board_callout(callout='init_fans', boost=DEFAULT_INIT_TRANSITIONAL)
Logger.warn("killed by signal %d" % (signum, ))
if signum == signal.SIGQUIT and wdfile:
Logger.info("Killed with SIGQUIT - stopping watchdog.")
stop_watchdog()
sys.exit('killed')
def get_fan_power_status(self):
'''
Method invokes board action to determine fan power status.
If not applicable returns True.
'''
if board_callout(callout='read_power'):
return True
return False
def update_zones(self, dead_fans, time_difference):
"""
TODO: Need to change logic here.
# Platforms with chassis_intrusion mode enabled
if chassis_intrusion:
set the chassis_intrusion_boost_flag to 0
and then do necessary checks to set flag to 1
if chassis_intrusion_boost_flag:
run boost mode
else:
run normal mode
else
# Platforms WITHOUT chassis_intrusion mode
run normal mode
"""
sensors_tuples = self.machine.read_sensors(self.sensors)
self.fsc_safe_guards(sensors_tuples)
for zone in self.zones:
Logger.info("PWM: %s" % (json.dumps(zone.pwm_output)))
chassis_intrusion_boost_flag = 0
if self.chassis_intrusion:
self_tray_pull_out = board_callout(
callout='chassis_intrusion')
if self_tray_pull_out == 1:
chassis_intrusion_boost_flag = 1
if chassis_intrusion_boost_flag == 0:
pwmval = zone.run(sensors=sensors_tuples, dt=time_difference)
else:
pwmval = self.boost
if self.fan_fail:
if self.boost_type == 'progressive' and self.fan_dead_boost:
# Cases where we want to progressively bump PWMs
dead = len(dead_fans)
if dead > 0:
Logger.info("Progressive mode: Failed fans: %s" %
(', '.join([str(i.label) for i in dead_fans],)))
for fan_count, rate in self.fan_dead_boost["data"]:
if dead <= fan_count:
pwmval = clamp(pwmval + (dead * rate), 0, 100)
break
else:
pwmval = self.boost
else:
if dead_fans:
# If not progressive ,when there is 1 fan failed, boost all fans
Logger.info("Failed fans: %s" % (
', '.join([str(i.label) for i in dead_fans],)))
pwmval = self.boost
if self.fan_dead_boost:
# If all the fans failed take action after a few cycles
if len(dead_fans) == len(self.fans):
self.all_fan_fail_counter = self.all_fan_fail_counter + 1
Logger.warn("Currently all fans failed for {} cycles".format(self.all_fan_fail_counter))
if self.fan_dead_boost["threshold"] and self.fan_dead_boost["action"]:
if self.all_fan_fail_counter >= self.fan_dead_boost["threshold"]:
self.fsc_host_action(
action=self.fan_dead_boost["action"],
cause="All fans are bad for more than "
+ str(self.fan_dead_boost["threshold"])
+ " cycles"
)
else:
# If atleast 1 fan is working reset the counter
self.all_fan_fail_counter = 0
if abs(zone.last_pwm - pwmval) > self.ramp_rate:
if pwmval < zone.last_pwm:
pwmval = zone.last_pwm - self.ramp_rate
else:
pwmval = zone.last_pwm + self.ramp_rate
zone.last_pwm = pwmval
if hasattr(zone.pwm_output, '__iter__'):
for output in zone.pwm_output:
self.machine.set_pwm(self.fans.get(
str(output)), pwmval)
else:
self.machine.set_pwm(self.fans[zone.pwm_output], pwmval)
global wdfile
board_callout(callout='init_fans', boost=DEFAULT_INIT_TRANSITIONAL)
Logger.warn("killed by signal %d" % (signum,))
if signum == signal.SIGQUIT and wdfile:
Logger.info("Killed with SIGQUIT - stopping watchdog.")
wdfile.write(b"X")
wdfile.flush()
wdfile.close()
wdfile = None
sys.exit('killed')
if __name__ == "__main__":
try:
signal.signal(signal.SIGTERM, handle_term)
signal.signal(signal.SIGINT, handle_term)
signal.signal(signal.SIGQUIT, handle_term)
if len(sys.argv) > 1:
llevel = sys.argv[1]
else:
llevel = 'warning'
fscd = Fscd(log_level=llevel)
fscd.run()
except Exception:
board_callout(callout='init_fans', boost=DEFAULT_INIT_TRANSITIONAL)
(etype, e) = sys.exc_info()[:2]
Logger.crit("failed, exception: " + str(etype))
traceback.print_exc()
for line in traceback.format_exc().split('\n'):
Logger.crit(line)
def update_zones(self, dead_fans, time_difference):
"""
TODO: Need to change logic here.
# Platforms with chassis_intrusion mode enabled
if chassis_intrusion:
set the chassis_intrusion_boost_flag to 0
and then do necessary checks to set flag to 1
if chassis_intrusion_boost_flag:
run boost mode
else:
run normal mode
else
# Platforms WITHOUT chassis_intrusion mode
run normal mode
# Platforms with enable_fsc_sensor_check mode enabled
if enable_fsc_sensor_check:
set the sensor_violated_flag to 0
and then do necessary checks to set flag to 1
if sensor_violated_flag:
run boost mode
else:
run normal mode
else
# Platforms WITHOUT enable_fsc_sensor_check mode
run normal mode
"""
ctx = {}
if not self.sensor_filter_all:
sensors_tuples = self.machine.read_sensors(self.sensors, None)
self.fsc_safe_guards(sensors_tuples)
for zone in self.zones:
if self.sensor_filter_all:
sensors_tuples = self.machine.read_sensors(
self.sensors, zone.expr_meta)
self.fsc_safe_guards(sensors_tuples)
Logger.info("PWM: %s" % (json.dumps(zone.pwm_output)))
mode = 0
chassis_intrusion_boost_flag = 0
sensor_violated_flag = 0
if self.chassis_intrusion:
self_tray_pull_out = board_callout(callout="chassis_intrusion")
if self_tray_pull_out == 1:
chassis_intrusion_boost_flag = 1
if self.enable_fsc_sensor_check:
Logger.info("enable_fsc_sensor_check")
if self.fsc_sensor_check(sensors_tuples) != 0:
sensor_violated_flag = 1
Logger.debug(" dead_fans(%d) " % len(dead_fans))
Logger.debug("Calculate")
if chassis_intrusion_boost_flag == 0 and sensor_violated_flag == 0:
ctx["dt"] = time_difference
ctx["dead_fans"] = dead_fans
ctx["last_pwm"] = zone.last_pwm
ignore_fan_mode = False
if self.non_fanfail_limited_boost and dead_fans:
ignore_fan_mode = True
pwmval = zone.run(sensors=sensors_tuples,
ctx=ctx,
ignore_mode=ignore_fan_mode)
mode = zone.get_set_fan_mode(mode, action="read")
# if we set pwm_sensor_boost_value option, assign it to pwmval
if self.pwm_sensor_boost_value != None and \
int(mode) == fan_mode["boost_mode"]:
if pwmval == self.boost:
pwmval = self.pwm_sensor_boost_value
else:
pwmval = self.boost
mode = fan_mode["boost_mode"]
if self.fan_fail:
boost_record_path = RECORD_DIR + "fan_fail_boost"
if self.boost_type == "progressive" and self.fan_dead_boost:
# Cases where we want to progressively bump PWMs
dead = len(dead_fans)
if dead > 0:
Logger.info(
"Progressive mode: Failed fans: %s" %
(", ".join([str(i.label) for i in dead_fans])))
for fan_count, rate in self.fan_dead_boost["data"]:
if dead <= fan_count:
pwmval = clamp(pwmval + (dead * rate), 0, 100)
mode = fan_mode["normal_mode"]
if os.path.isfile(boost_record_path):
os.remove(boost_record_path)
break
else:
pwmval = self.boost
mode = fan_mode["boost_mode"]
if not os.path.isfile(boost_record_path):
fan_fail_boost_record = open(
boost_record_path, "w")
fan_fail_boost_record.close()
else:
if os.path.isfile(boost_record_path):
os.remove(boost_record_path)
else:
if dead_fans:
# If not progressive ,when there is 1 fan failed, boost all fans
Logger.info(
"Failed fans: %s" %
(", ".join([str(i.label) for i in dead_fans])))
# choose the higher PWM
if self.output_max_boost_pwm:
pwmval = self.boost if pwmval < self.boost else pwmval
else:
pwmval = self.boost
mode = fan_mode["boost_mode"]
if not os.path.isfile(boost_record_path):
fan_fail_boost_record = open(
boost_record_path, "w")
fan_fail_boost_record.close()
else:
if os.path.isfile(boost_record_path):
os.remove(boost_record_path)
if self.fan_dead_boost:
# If all the fans failed take action after a few cycles
if len(dead_fans) == len(self.fans):
self.all_fan_fail_counter = self.all_fan_fail_counter + 1
Logger.warn(
"Currently all fans failed for {} cycles".
format(self.all_fan_fail_counter))
if (self.fan_dead_boost["threshold"]
and self.fan_dead_boost["action"]):
if (self.all_fan_fail_counter >=
self.fan_dead_boost["threshold"]):
self.fsc_host_action(
action=self.fan_dead_boost["action"],
cause="All fans are bad for more than "
+
str(self.fan_dead_boost["threshold"]) +
" cycles",
)
else:
# If atleast 1 fan is working reset the counter
self.all_fan_fail_counter = 0
if self.fan_limit_upper_pwm:
if pwmval > self.fan_limit_upper_pwm:
pwmval = self.fan_limit_upper_pwm
if self.fan_limit_lower_pwm:
if pwmval < self.fan_limit_lower_pwm:
pwmval = self.fan_limit_lower_pwm
# if no fan fail, the max of pwm is non_fanfail_limited_boost pwm:
if self.non_fanfail_limited_boost and not dead_fans:
pwmval = clamp(pwmval, 0, self.non_fanfail_limited_boost)
if abs(zone.last_pwm - pwmval) > self.ramp_rate:
if pwmval < zone.last_pwm:
pwmval = zone.last_pwm - self.ramp_rate
else:
pwmval = zone.last_pwm + self.ramp_rate
zone.last_pwm = pwmval
if hasattr(zone.pwm_output, "__iter__"):
for output in zone.pwm_output:
self.machine.set_pwm(self.fans.get(str(output)), pwmval)
else:
self.machine.set_pwm(self.fans[zone.pwm_output], pwmval)
zone.get_set_fan_mode(mode, action="write")
