def _update_alerting_temps(
self, temps: Mapping[TempName, Optional[TempStatus]]) -> None:
stopped_alerting_temps = self._alerting_temps.copy()
for name, status in temps.items():
temp_alerting_reason = self._temp_alerting_reason(status)
if not temp_alerting_reason:
continue
if name in self._alerting_temps:
# Still alerting
stopped_alerting_temps.discard(name)
continue
# Just started alerting
self._alerting_temps.add(name)
logger.warning(
"%s started on temp. name: %s, status: %s, reason: %s",
self.trigger_name.upper(),
name,
status,
temp_alerting_reason,
)
self._alert_cmd(self.temp_commands[name].enter_cmd)
for name in stopped_alerting_temps:
self._alerting_temps.discard(name)
status = temps[name]
logger.warning(
"%s ended on temp: name: %s, status: %s",
self.trigger_name.upper(),
name,
status,
)
self._alert_cmd(self.temp_commands[name].leave_cmd)
def exec_shell_command(shell_command: str, timeout: int = 5) -> str:
try:
p = subprocess.run(
shell_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
check=True,
timeout=timeout,
)
out = p.stdout.decode("ascii")
err = p.stderr.decode().strip()
if err:
logger.warning(
"Shell command '%s' executed successfully, but printed to stderr:\n%s",
shell_command,
err,
)
return out
except subprocess.CalledProcessError as e:
ec = e.returncode
out = e.stdout.decode().strip()
err = e.stderr.decode().strip()
logger.error(
"Shell command '%s' failed (exit code %s):\nstdout:\n%s\nstderr:\n%s\n",
shell_command,
ec,
out,
err,
)
raise
def _incoming_message(self, message: Dict[str, Any]) -> None:
# Called by the pyserial Protocol `_StatusProtocol`.
if "error" in message:
logger.warning("Received an error from Arduino %s: %r", self.url,
message)
else:
self._update_status(message)
def set_all_to_full_speed(self) -> None:
for name, fan in self.fans.items():
if name in self._failed_fans:
continue
try:
fan.set_full_speed()
except Exception as e:
logger.warning("Unable to set the fan '%s' to full speed:\n%s",
name, e)
def report(self, reason: str, message: str) -> None:
logger.info("[REPORT] Reason: %s. Message: %s", reason, message)
try:
rc = self._report_command
rc = rc.replace("%REASON%", reason)
rc = rc.replace("%MESSAGE%", message)
exec_shell_command(rc)
except Exception as ex:
logger.warning("Report failed: %s", ex, exc_info=True)
def _alert_cmd(self, shell_cmd):
if not shell_cmd:
return
try:
exec_shell_command(shell_cmd)
except Exception as e:
logger.warning(
"Enable to execute %s trigger command %s:\n%s",
self.trigger_name,
shell_cmd,
e,
)
def _get_temps(self) -> Mapping[TempName, Optional[TempStatus]]:
result = {}
for name, temp in self.temps.items():
try:
status = temp.get() # type: Optional[TempStatus]
except Exception as e:
status = None
logger.warning("Temp sensor [%s] has failed: %s",
name,
e,
exc_info=True)
else:
logger.debug("Temp status [%s]: %s", name, status)
result[name] = status
return result
def tick(self) -> None:
with self.metrics.measure_tick():
temps = self._get_temps()
self.fans.check_speeds()
self.triggers.check(temps)
if self.triggers.is_alerting:
self.fans.set_all_to_full_speed()
else:
speeds = self._map_temps_to_fan_speeds(temps)
self.fans.set_fan_speeds(speeds)
try:
self.metrics.tick(temps, self.fans, self.triggers)
except Exception:
logger.warning("Failed to collect metrics", exc_info=True)
def _collect_fan_metrics(self, fans, fan_name, pwm_fan_norm):
self.fan_pwm_line_start.labels(fan_name).set(
pwm_fan_norm.pwm_line_start)
self.fan_pwm_line_end.labels(fan_name).set(pwm_fan_norm.pwm_line_end)
self.fan_is_stopped.labels(fan_name).set(fans.is_fan_stopped(fan_name))
self.fan_is_failing.labels(fan_name).set(fans.is_fan_failing(fan_name))
try:
self.fan_rpm.labels(fan_name).set(pwm_fan_norm.get_speed())
self.fan_pwm.labels(fan_name).set(pwm_fan_norm.get_raw())
self.fan_pwm_normalized.labels(fan_name).set(pwm_fan_norm.get())
except Exception:
logger.warning("Failed to collect metrics for fan %s",
fan_name,
exc_info=True)
self.fan_rpm.labels(fan_name).set(none_to_nan(None))
self.fan_pwm.labels(fan_name).set(none_to_nan(None))
self.fan_pwm_normalized.labels(fan_name).set(none_to_nan(None))
def set_fan_speeds(self, speeds: Mapping[FanName, PWMValueNorm]) -> None:
assert speeds.keys() == self.fans.keys()
self._stopped_fans.clear()
for name, pwm_norm in speeds.items():
fan = self.fans[name]
assert 0.0 <= pwm_norm <= 1.0
if name in self._failed_fans:
continue
try:
pwm = fan.set(pwm_norm)
except Exception as e:
logger.warning("Unable to set the fan '%s' to speed %s:\n%s",
name, pwm_norm, e)
else:
logger.debug("Fan status [%s]: speed: %.3f, pwm: %s", name,
pwm_norm, pwm)
if fan.is_pwm_stopped(pwm):
self._stopped_fans.add(name)
def _collect_any_fan_metrics(
self,
fans: Fans,
fan_name: AnyFanName,
pwm_fan_norm: Union[PWMFanNorm, ReadonlyPWMFanNorm],
):
self.fan_is_stopped.labels(fan_name).set(fans.is_fan_stopped(fan_name))
self.fan_is_failing.labels(fan_name).set(fans.is_fan_failing(fan_name))
try:
self.fan_rpm.labels(fan_name).set(pwm_fan_norm.get_speed())
self.fan_pwm.labels(fan_name).set(
none_to_nan(pwm_fan_norm.get_raw()))
self.fan_pwm_normalized.labels(fan_name).set(
none_to_nan(pwm_fan_norm.get()))
except Exception:
logger.warning("Failed to collect metrics for fan %s",
fan_name,
exc_info=True)
self.fan_rpm.labels(fan_name).set(none_to_nan(None))
self.fan_pwm.labels(fan_name).set(none_to_nan(None))
self.fan_pwm_normalized.labels(fan_name).set(none_to_nan(None))
def _parse_mappings(
config: configparser.ConfigParser,
fans: Mapping[FanName, PWMFanNorm],
temps: Mapping[TempName, FilteredTemp],
) -> Mapping[MappingName, FansTempsRelation]:
mappings: Dict[MappingName, FansTempsRelation] = {}
for section in iter_sections(config, "mapping", MappingName):
# temps:
mapping_temps = [
TempName(temp_name.strip())
for temp_name in section["temps"].split(",")
]
mapping_temps = [s for s in mapping_temps if s]
if not mapping_temps:
raise RuntimeError("Temps must not be empty in the '%s' mapping" %
section.name)
for temp_name in mapping_temps:
if temp_name not in temps:
raise RuntimeError("Unknown temp '%s' in mapping '%s'" %
(temp_name, section.name))
if len(mapping_temps) != len(set(mapping_temps)):
raise RuntimeError("There are duplicate temps in mapping '%s'" %
section.name)
# fans:
fans_with_speed = [
fan_with_speed.strip()
for fan_with_speed in section["fans"].split(",")
]
fans_with_speed = [s for s in fans_with_speed if s]
fan_speed_pairs = [
fan_with_speed.split("*") for fan_with_speed in fans_with_speed
]
for fan_speed_pair in fan_speed_pairs:
if len(fan_speed_pair) not in (1, 2):
raise RuntimeError(
"Invalid fan specification '%s' in mapping '%s'" %
(fan_speed_pair, section.name))
mapping_fans = [
FanSpeedModifier(
fan=FanName(fan_speed_pair[0].strip()),
modifier=(float(fan_speed_pair[1].strip(
) if len(fan_speed_pair) == 2 else 1.0)),
) for fan_speed_pair in fan_speed_pairs
]
for fan_speed_modifier in mapping_fans:
if fan_speed_modifier.fan not in fans:
raise RuntimeError("Unknown fan '%s' in mapping '%s'" %
(fan_speed_modifier.fan, section.name))
if not (0 < fan_speed_modifier.modifier <= 1.0):
raise RuntimeError(
"Invalid fan modifier '%s' in mapping '%s' for fan '%s': "
"the allowed range is (0.0;1.0]." % (
fan_speed_modifier.modifier,
section.name,
fan_speed_modifier.fan,
))
if len(mapping_fans) != len(
set(fan_speed_modifier.fan
for fan_speed_modifier in mapping_fans)):
raise RuntimeError("There are duplicate fans in mapping '%s'" %
section.name)
if section.name in mappings:
raise RuntimeError(
"Duplicate mapping section declaration for '%s'" %
section.name)
mappings[section.name] = FansTempsRelation(temps=mapping_temps,
fans=mapping_fans)
section.ensure_no_unused_keys()
unused_temps = set(temps.keys())
unused_fans = set(fans.keys())
for relation in mappings.values():
unused_temps -= set(relation.temps)
unused_fans -= set(fan_speed_modifier.fan
for fan_speed_modifier in relation.fans)
if unused_temps:
logger.warning(
"The following temps are defined but not used in any mapping: %s",
unused_temps,
)
if unused_fans:
raise RuntimeError(
"The following fans are defined but not used in any mapping: %s" %
unused_fans)
return mappings
