def handle_term(signum, frame): global wdfile board_callout(callout='init_fans', boost=DEFAULT_INIT_TRANSITIONAL) Logger.warn("killed by signal %d" % (signum, )) if signum == signal.SIGQUIT and wdfile: Logger.info("Killed with SIGQUIT - stopping watchdog.") stop_watchdog() sys.exit('killed')
def get_fan_power_status(self): ''' Method invokes board action to determine fan power status. If not applicable returns True. ''' if board_callout(callout='read_power'): return True return False
def update_zones(self, dead_fans, time_difference): """ TODO: Need to change logic here. # Platforms with chassis_intrusion mode enabled if chassis_intrusion: set the chassis_intrusion_boost_flag to 0 and then do necessary checks to set flag to 1 if chassis_intrusion_boost_flag: run boost mode else: run normal mode else # Platforms WITHOUT chassis_intrusion mode run normal mode """ sensors_tuples = self.machine.read_sensors(self.sensors) self.fsc_safe_guards(sensors_tuples) for zone in self.zones: Logger.info("PWM: %s" % (json.dumps(zone.pwm_output))) chassis_intrusion_boost_flag = 0 if self.chassis_intrusion: self_tray_pull_out = board_callout( callout='chassis_intrusion') if self_tray_pull_out == 1: chassis_intrusion_boost_flag = 1 if chassis_intrusion_boost_flag == 0: pwmval = zone.run(sensors=sensors_tuples, dt=time_difference) else: pwmval = self.boost if self.fan_fail: if self.boost_type == 'progressive' and self.fan_dead_boost: # Cases where we want to progressively bump PWMs dead = len(dead_fans) if dead > 0: Logger.info("Progressive mode: Failed fans: %s" % (', '.join([str(i.label) for i in dead_fans],))) for fan_count, rate in self.fan_dead_boost["data"]: if dead <= fan_count: pwmval = clamp(pwmval + (dead * rate), 0, 100) break else: pwmval = self.boost else: if dead_fans: # If not progressive ,when there is 1 fan failed, boost all fans Logger.info("Failed fans: %s" % ( ', '.join([str(i.label) for i in dead_fans],))) pwmval = self.boost if self.fan_dead_boost: # If all the fans failed take action after a few cycles if len(dead_fans) == len(self.fans): self.all_fan_fail_counter = self.all_fan_fail_counter + 1 Logger.warn("Currently all fans failed for {} cycles".format(self.all_fan_fail_counter)) if self.fan_dead_boost["threshold"] and self.fan_dead_boost["action"]: if self.all_fan_fail_counter >= self.fan_dead_boost["threshold"]: self.fsc_host_action( action=self.fan_dead_boost["action"], cause="All fans are bad for more than " + str(self.fan_dead_boost["threshold"]) + " cycles" ) else: # If atleast 1 fan is working reset the counter self.all_fan_fail_counter = 0 if abs(zone.last_pwm - pwmval) > self.ramp_rate: if pwmval < zone.last_pwm: pwmval = zone.last_pwm - self.ramp_rate else: pwmval = zone.last_pwm + self.ramp_rate zone.last_pwm = pwmval if hasattr(zone.pwm_output, '__iter__'): for output in zone.pwm_output: self.machine.set_pwm(self.fans.get( str(output)), pwmval) else: self.machine.set_pwm(self.fans[zone.pwm_output], pwmval)
global wdfile board_callout(callout='init_fans', boost=DEFAULT_INIT_TRANSITIONAL) Logger.warn("killed by signal %d" % (signum,)) if signum == signal.SIGQUIT and wdfile: Logger.info("Killed with SIGQUIT - stopping watchdog.") wdfile.write(b"X") wdfile.flush() wdfile.close() wdfile = None sys.exit('killed') if __name__ == "__main__": try: signal.signal(signal.SIGTERM, handle_term) signal.signal(signal.SIGINT, handle_term) signal.signal(signal.SIGQUIT, handle_term) if len(sys.argv) > 1: llevel = sys.argv[1] else: llevel = 'warning' fscd = Fscd(log_level=llevel) fscd.run() except Exception: board_callout(callout='init_fans', boost=DEFAULT_INIT_TRANSITIONAL) (etype, e) = sys.exc_info()[:2] Logger.crit("failed, exception: " + str(etype)) traceback.print_exc() for line in traceback.format_exc().split('\n'): Logger.crit(line)
def update_zones(self, dead_fans, time_difference): """ TODO: Need to change logic here. # Platforms with chassis_intrusion mode enabled if chassis_intrusion: set the chassis_intrusion_boost_flag to 0 and then do necessary checks to set flag to 1 if chassis_intrusion_boost_flag: run boost mode else: run normal mode else # Platforms WITHOUT chassis_intrusion mode run normal mode # Platforms with enable_fsc_sensor_check mode enabled if enable_fsc_sensor_check: set the sensor_violated_flag to 0 and then do necessary checks to set flag to 1 if sensor_violated_flag: run boost mode else: run normal mode else # Platforms WITHOUT enable_fsc_sensor_check mode run normal mode """ ctx = {} if not self.sensor_filter_all: sensors_tuples = self.machine.read_sensors(self.sensors, None) self.fsc_safe_guards(sensors_tuples) for zone in self.zones: if self.sensor_filter_all: sensors_tuples = self.machine.read_sensors( self.sensors, zone.expr_meta) self.fsc_safe_guards(sensors_tuples) Logger.info("PWM: %s" % (json.dumps(zone.pwm_output))) mode = 0 chassis_intrusion_boost_flag = 0 sensor_violated_flag = 0 if self.chassis_intrusion: self_tray_pull_out = board_callout(callout="chassis_intrusion") if self_tray_pull_out == 1: chassis_intrusion_boost_flag = 1 if self.enable_fsc_sensor_check: Logger.info("enable_fsc_sensor_check") if self.fsc_sensor_check(sensors_tuples) != 0: sensor_violated_flag = 1 Logger.debug(" dead_fans(%d) " % len(dead_fans)) Logger.debug("Calculate") if chassis_intrusion_boost_flag == 0 and sensor_violated_flag == 0: ctx["dt"] = time_difference ctx["dead_fans"] = dead_fans ctx["last_pwm"] = zone.last_pwm ignore_fan_mode = False if self.non_fanfail_limited_boost and dead_fans: ignore_fan_mode = True pwmval = zone.run(sensors=sensors_tuples, ctx=ctx, ignore_mode=ignore_fan_mode) mode = zone.get_set_fan_mode(mode, action="read") # if we set pwm_sensor_boost_value option, assign it to pwmval if self.pwm_sensor_boost_value != None and \ int(mode) == fan_mode["boost_mode"]: if pwmval == self.boost: pwmval = self.pwm_sensor_boost_value else: pwmval = self.boost mode = fan_mode["boost_mode"] if self.fan_fail: boost_record_path = RECORD_DIR + "fan_fail_boost" if self.boost_type == "progressive" and self.fan_dead_boost: # Cases where we want to progressively bump PWMs dead = len(dead_fans) if dead > 0: Logger.info( "Progressive mode: Failed fans: %s" % (", ".join([str(i.label) for i in dead_fans]))) for fan_count, rate in self.fan_dead_boost["data"]: if dead <= fan_count: pwmval = clamp(pwmval + (dead * rate), 0, 100) mode = fan_mode["normal_mode"] if os.path.isfile(boost_record_path): os.remove(boost_record_path) break else: pwmval = self.boost mode = fan_mode["boost_mode"] if not os.path.isfile(boost_record_path): fan_fail_boost_record = open( boost_record_path, "w") fan_fail_boost_record.close() else: if os.path.isfile(boost_record_path): os.remove(boost_record_path) else: if dead_fans: # If not progressive ,when there is 1 fan failed, boost all fans Logger.info( "Failed fans: %s" % (", ".join([str(i.label) for i in dead_fans]))) # choose the higher PWM if self.output_max_boost_pwm: pwmval = self.boost if pwmval < self.boost else pwmval else: pwmval = self.boost mode = fan_mode["boost_mode"] if not os.path.isfile(boost_record_path): fan_fail_boost_record = open( boost_record_path, "w") fan_fail_boost_record.close() else: if os.path.isfile(boost_record_path): os.remove(boost_record_path) if self.fan_dead_boost: # If all the fans failed take action after a few cycles if len(dead_fans) == len(self.fans): self.all_fan_fail_counter = self.all_fan_fail_counter + 1 Logger.warn( "Currently all fans failed for {} cycles". format(self.all_fan_fail_counter)) if (self.fan_dead_boost["threshold"] and self.fan_dead_boost["action"]): if (self.all_fan_fail_counter >= self.fan_dead_boost["threshold"]): self.fsc_host_action( action=self.fan_dead_boost["action"], cause="All fans are bad for more than " + str(self.fan_dead_boost["threshold"]) + " cycles", ) else: # If atleast 1 fan is working reset the counter self.all_fan_fail_counter = 0 if self.fan_limit_upper_pwm: if pwmval > self.fan_limit_upper_pwm: pwmval = self.fan_limit_upper_pwm if self.fan_limit_lower_pwm: if pwmval < self.fan_limit_lower_pwm: pwmval = self.fan_limit_lower_pwm # if no fan fail, the max of pwm is non_fanfail_limited_boost pwm: if self.non_fanfail_limited_boost and not dead_fans: pwmval = clamp(pwmval, 0, self.non_fanfail_limited_boost) if abs(zone.last_pwm - pwmval) > self.ramp_rate: if pwmval < zone.last_pwm: pwmval = zone.last_pwm - self.ramp_rate else: pwmval = zone.last_pwm + self.ramp_rate zone.last_pwm = pwmval if hasattr(zone.pwm_output, "__iter__"): for output in zone.pwm_output: self.machine.set_pwm(self.fans.get(str(output)), pwmval) else: self.machine.set_pwm(self.fans[zone.pwm_output], pwmval) zone.get_set_fan_mode(mode, action="write")