def _update_alerting_temps( self, temps: Mapping[TempName, Optional[TempStatus]]) -> None: stopped_alerting_temps = self._alerting_temps.copy() for name, status in temps.items(): temp_alerting_reason = self._temp_alerting_reason(status) if not temp_alerting_reason: continue if name in self._alerting_temps: # Still alerting stopped_alerting_temps.discard(name) continue # Just started alerting self._alerting_temps.add(name) logger.warning( "%s started on temp. name: %s, status: %s, reason: %s", self.trigger_name.upper(), name, status, temp_alerting_reason, ) self._alert_cmd(self.temp_commands[name].enter_cmd) for name in stopped_alerting_temps: self._alerting_temps.discard(name) status = temps[name] logger.warning( "%s ended on temp: name: %s, status: %s", self.trigger_name.upper(), name, status, ) self._alert_cmd(self.temp_commands[name].leave_cmd)
def exec_shell_command(shell_command: str, timeout: int = 5) -> str: try: p = subprocess.run( shell_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, check=True, timeout=timeout, ) out = p.stdout.decode("ascii") err = p.stderr.decode().strip() if err: logger.warning( "Shell command '%s' executed successfully, but printed to stderr:\n%s", shell_command, err, ) return out except subprocess.CalledProcessError as e: ec = e.returncode out = e.stdout.decode().strip() err = e.stderr.decode().strip() logger.error( "Shell command '%s' failed (exit code %s):\nstdout:\n%s\nstderr:\n%s\n", shell_command, ec, out, err, ) raise
def _incoming_message(self, message: Dict[str, Any]) -> None: # Called by the pyserial Protocol `_StatusProtocol`. if "error" in message: logger.warning("Received an error from Arduino %s: %r", self.url, message) else: self._update_status(message)
def set_all_to_full_speed(self) -> None: for name, fan in self.fans.items(): if name in self._failed_fans: continue try: fan.set_full_speed() except Exception as e: logger.warning("Unable to set the fan '%s' to full speed:\n%s", name, e)
def report(self, reason: str, message: str) -> None: logger.info("[REPORT] Reason: %s. Message: %s", reason, message) try: rc = self._report_command rc = rc.replace("%REASON%", reason) rc = rc.replace("%MESSAGE%", message) exec_shell_command(rc) except Exception as ex: logger.warning("Report failed: %s", ex, exc_info=True)
def _alert_cmd(self, shell_cmd): if not shell_cmd: return try: exec_shell_command(shell_cmd) except Exception as e: logger.warning( "Enable to execute %s trigger command %s:\n%s", self.trigger_name, shell_cmd, e, )
def _get_temps(self) -> Mapping[TempName, Optional[TempStatus]]: result = {} for name, temp in self.temps.items(): try: status = temp.get() # type: Optional[TempStatus] except Exception as e: status = None logger.warning("Temp sensor [%s] has failed: %s", name, e, exc_info=True) else: logger.debug("Temp status [%s]: %s", name, status) result[name] = status return result
def tick(self) -> None: with self.metrics.measure_tick(): temps = self._get_temps() self.fans.check_speeds() self.triggers.check(temps) if self.triggers.is_alerting: self.fans.set_all_to_full_speed() else: speeds = self._map_temps_to_fan_speeds(temps) self.fans.set_fan_speeds(speeds) try: self.metrics.tick(temps, self.fans, self.triggers) except Exception: logger.warning("Failed to collect metrics", exc_info=True)
def _collect_fan_metrics(self, fans, fan_name, pwm_fan_norm): self.fan_pwm_line_start.labels(fan_name).set( pwm_fan_norm.pwm_line_start) self.fan_pwm_line_end.labels(fan_name).set(pwm_fan_norm.pwm_line_end) self.fan_is_stopped.labels(fan_name).set(fans.is_fan_stopped(fan_name)) self.fan_is_failing.labels(fan_name).set(fans.is_fan_failing(fan_name)) try: self.fan_rpm.labels(fan_name).set(pwm_fan_norm.get_speed()) self.fan_pwm.labels(fan_name).set(pwm_fan_norm.get_raw()) self.fan_pwm_normalized.labels(fan_name).set(pwm_fan_norm.get()) except Exception: logger.warning("Failed to collect metrics for fan %s", fan_name, exc_info=True) self.fan_rpm.labels(fan_name).set(none_to_nan(None)) self.fan_pwm.labels(fan_name).set(none_to_nan(None)) self.fan_pwm_normalized.labels(fan_name).set(none_to_nan(None))
def set_fan_speeds(self, speeds: Mapping[FanName, PWMValueNorm]) -> None: assert speeds.keys() == self.fans.keys() self._stopped_fans.clear() for name, pwm_norm in speeds.items(): fan = self.fans[name] assert 0.0 <= pwm_norm <= 1.0 if name in self._failed_fans: continue try: pwm = fan.set(pwm_norm) except Exception as e: logger.warning("Unable to set the fan '%s' to speed %s:\n%s", name, pwm_norm, e) else: logger.debug("Fan status [%s]: speed: %.3f, pwm: %s", name, pwm_norm, pwm) if fan.is_pwm_stopped(pwm): self._stopped_fans.add(name)
def _collect_any_fan_metrics( self, fans: Fans, fan_name: AnyFanName, pwm_fan_norm: Union[PWMFanNorm, ReadonlyPWMFanNorm], ): self.fan_is_stopped.labels(fan_name).set(fans.is_fan_stopped(fan_name)) self.fan_is_failing.labels(fan_name).set(fans.is_fan_failing(fan_name)) try: self.fan_rpm.labels(fan_name).set(pwm_fan_norm.get_speed()) self.fan_pwm.labels(fan_name).set( none_to_nan(pwm_fan_norm.get_raw())) self.fan_pwm_normalized.labels(fan_name).set( none_to_nan(pwm_fan_norm.get())) except Exception: logger.warning("Failed to collect metrics for fan %s", fan_name, exc_info=True) self.fan_rpm.labels(fan_name).set(none_to_nan(None)) self.fan_pwm.labels(fan_name).set(none_to_nan(None)) self.fan_pwm_normalized.labels(fan_name).set(none_to_nan(None))
def _parse_mappings( config: configparser.ConfigParser, fans: Mapping[FanName, PWMFanNorm], temps: Mapping[TempName, FilteredTemp], ) -> Mapping[MappingName, FansTempsRelation]: mappings: Dict[MappingName, FansTempsRelation] = {} for section in iter_sections(config, "mapping", MappingName): # temps: mapping_temps = [ TempName(temp_name.strip()) for temp_name in section["temps"].split(",") ] mapping_temps = [s for s in mapping_temps if s] if not mapping_temps: raise RuntimeError("Temps must not be empty in the '%s' mapping" % section.name) for temp_name in mapping_temps: if temp_name not in temps: raise RuntimeError("Unknown temp '%s' in mapping '%s'" % (temp_name, section.name)) if len(mapping_temps) != len(set(mapping_temps)): raise RuntimeError("There are duplicate temps in mapping '%s'" % section.name) # fans: fans_with_speed = [ fan_with_speed.strip() for fan_with_speed in section["fans"].split(",") ] fans_with_speed = [s for s in fans_with_speed if s] fan_speed_pairs = [ fan_with_speed.split("*") for fan_with_speed in fans_with_speed ] for fan_speed_pair in fan_speed_pairs: if len(fan_speed_pair) not in (1, 2): raise RuntimeError( "Invalid fan specification '%s' in mapping '%s'" % (fan_speed_pair, section.name)) mapping_fans = [ FanSpeedModifier( fan=FanName(fan_speed_pair[0].strip()), modifier=(float(fan_speed_pair[1].strip( ) if len(fan_speed_pair) == 2 else 1.0)), ) for fan_speed_pair in fan_speed_pairs ] for fan_speed_modifier in mapping_fans: if fan_speed_modifier.fan not in fans: raise RuntimeError("Unknown fan '%s' in mapping '%s'" % (fan_speed_modifier.fan, section.name)) if not (0 < fan_speed_modifier.modifier <= 1.0): raise RuntimeError( "Invalid fan modifier '%s' in mapping '%s' for fan '%s': " "the allowed range is (0.0;1.0]." % ( fan_speed_modifier.modifier, section.name, fan_speed_modifier.fan, )) if len(mapping_fans) != len( set(fan_speed_modifier.fan for fan_speed_modifier in mapping_fans)): raise RuntimeError("There are duplicate fans in mapping '%s'" % section.name) if section.name in mappings: raise RuntimeError( "Duplicate mapping section declaration for '%s'" % section.name) mappings[section.name] = FansTempsRelation(temps=mapping_temps, fans=mapping_fans) section.ensure_no_unused_keys() unused_temps = set(temps.keys()) unused_fans = set(fans.keys()) for relation in mappings.values(): unused_temps -= set(relation.temps) unused_fans -= set(fan_speed_modifier.fan for fan_speed_modifier in relation.fans) if unused_temps: logger.warning( "The following temps are defined but not used in any mapping: %s", unused_temps, ) if unused_fans: raise RuntimeError( "The following fans are defined but not used in any mapping: %s" % unused_fans) return mappings