class SelfTestAlertSource(AlertSource): level = AlertLevel.WARNING title = "Self-test error" schedule = IntervalSchedule(timedelta(minutes=5)) async def check(self): alerts = [] regexp = re.compile(r"\[(.*)\] (.*)") if os.path.exists(ALERT_FILE): with open(ALERT_FILE) as f: for line in f: line = line.rstrip() # Line looks like [PASS|FAIL]<text>, maybe other tags match = regexp.match(line) level = AlertLevel.WARNING if match: if match.group(1) in (TEST_WARNING): level = AlertLevel.WARNING elif match.group(1) in (TEST_FAIL, TEST_CRITICAL): level = AlertLevel.WARNING elif match.group(1) in (TEST_PASS): continue alerts.append(Alert(match.group(2), level=level)) else: alerts.append(Alert(line, level=level)) return alerts
class UpdateFailedAlertSource(FilePresenceAlertSource): level = AlertLevel.CRITICAL title = "Update failed. Check /data/update.failed for further details" schedule = IntervalSchedule(timedelta(hours=1)) path = "/data/update.failed"
class VolumeVersionAlertSource(ThreadedAlertSource): level = AlertLevel.WARNING title = "ZFS version is out of date" schedule = IntervalSchedule(timedelta(minutes=5)) def check_sync(self): alerts = [] for pool in self.middleware.call_sync("pool.query"): if not self.is_upgraded(pool): alerts.append(Alert( "New feature flags are available for volume %s. Refer " "to the \"Upgrading a ZFS Pool\" section of the User " "Guide for instructions.", pool["name"], )) proc = subprocess.Popen( "zfs upgrade | grep FILESYSTEM", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf8", ) output = proc.communicate()[0].strip(" ").strip("\n") if output: alerts.append(Alert( "ZFS filesystem version is out of date. Consider upgrading" " using \"zfs upgrade\" command line." )) return alerts def is_upgraded(self, pool): if not pool["is_decrypted"]: return True try: version = self.middleware.call_sync("notifier.zpool_version", pool["name"]) except ValueError: return True if version == "-": proc = subprocess.Popen([ "zpool", "get", "-H", "-o", "property,value", "all", pool["name"], ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf8") data = proc.communicate()[0].strip("\n") for line in data.split("\n"): if not line.startswith("feature") or "\t" not in line: continue prop, value = line.split("\t", 1) if value not in ("active", "enabled"): return False return True return False
class HasUpdateAlertSource(ThreadedAlertSource): level = AlertLevel.INFO title = "There is a new update available" schedule = IntervalSchedule(timedelta(hours=1)) def check_sync(self): try: self.middleware.call_sync("datastore.query", "system.update", None, {"get": True}) except IndexError: self.middleware.call_sync("datastore.insert", "system.update", { "upd_autocheck": True, "upd_train": "", }) path = self.middleware.call_sync("update.get_update_location") if not path: return try: updates = PendingUpdates(path) except Exception: updates = None if updates: return Alert("There is a new update available! Apply it in System -> Update tab.")
class IPMISELSpaceLeftAlertSource(AlertSource): schedule = IntervalSchedule(timedelta(minutes=5)) async def check(self): if not has_ipmi(): return return self._produce_alert_for_ipmitool_output(await ipmitool( ["sel", "info"])) def _produce_alert_for_ipmitool_output(self, output): sel_information = parse_sel_information(output) try: percent_used = int(sel_information["Percent Used"].rstrip("%")) except ValueError: return if percent_used > 90: return Alert( IPMISELSpaceLeftAlertClass, { "free": sel_information["Free Space"], "used": sel_information["Percent Used"], }, key=None, )
class HTTPD_SSL_AlertSource(AlertSource): level = AlertLevel.WARNING title = "FreeNAS HTTP server SSL misconfiguration" schedule = IntervalSchedule(timedelta(minutes=5)) async def check(self): alerts = [] if os.path.exists("/tmp/alert_invalid_ssl_nginx"): alerts.append( Alert( "FreeNAS does not support certificates with keys shorter than 1024 bits. " "HTTPS will not be enabled until a certificate having at least 1024 bit " "keylength is provided", )) for cert_name in glob.glob("/var/tmp/alert_invalidcert_*"): alerts.append( Alert( "The Certificate: %(cert_name)s is either malformed " "or invalid and cannot be used for any services. " "This Alert will remain here until the certificate is deleted", {"cert_name": cert_name.split("_", 2)[-1]}, )) for CA_name in glob.glob("/var/tmp/alert_invalidCA_*"): alerts.append( Alert( "The Certificate Authority(CA): %(CA_name)s is either " "malformed or invalid and cannot be used for any services. " "This Alert will remain here until the CA is deleted", {"CA_name": CA_name.split("_", 2)[-1]}, )) return alerts
class VolumeVersionAlertSource(ThreadedAlertSource): level = AlertLevel.WARNING title = "ZFS version is out of date" schedule = IntervalSchedule(timedelta(minutes=5)) def check_sync(self): alerts = [] for pool in self.middleware.call_sync("pool.query"): if not self.middleware.call_sync('pool.is_upgraded', pool["id"]): alerts.append( Alert( "New feature flags are available for volume %s. Refer " "to the \"Upgrading a ZFS Pool\" subsection in the " "User Guide \"Installing and Upgrading\" chapter " "and \"Upgrading\" section for more instructions.", pool["name"], )) proc = subprocess.Popen( "zfs upgrade | grep FILESYSTEM", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf8", ) output = proc.communicate()[0].strip(" ").strip("\n") if output: alerts.append( Alert( "ZFS filesystem version is out of date. Consider upgrading" " using \"zfs upgrade\" command line.")) return alerts
class NTPHealthCheckAlertSource(AlertSource): schedule = IntervalSchedule(timedelta(hours=12)) run_on_backup_node = False async def check(self): uptime_seconds = time.clock_gettime(time.CLOCK_MONOTONIC_RAW) if uptime_seconds < 300: return try: peers = await self.middleware.call("system.ntpserver.peers") except Exception: self.middleware.logger.warning("Failed to retrieve peers.", exc_info=True) peers = [] if not peers: return active_peer = [x for x in peers if x['status'].endswith('PEER')] if not active_peer: return Alert( NTPHealthCheckAlertClass, {'reason': f'No NTP peers: {[{x["remote"]: x["status"]} for x in peers]}'} ) peer = active_peer[0] if peer['offset'] < 300000: return return Alert( NTPHealthCheckAlertClass, {'reason': f'{peer["remote"]} has an offset of {peer["offset"]}, which exceeds permitted value of 5 minutes.'} )
class IPMISELAlertSource(AlertSource, DismissableAlertSource): level = AlertLevel.WARNING title = "IPMI System Event" schedule = IntervalSchedule(timedelta(minutes=5)) dismissed_datetime_kv_key = "alert:ipmi_sel:dismissed_datetime" async def check(self): if not has_ipmi(): return return await self._produce_alerts_for_ipmitool_output( (await run(["ipmitool", "-c", "sel", "elist"], encoding="utf8")).stdout) async def dismiss(self, alerts): await self.middleware.call("keyvalue.set", self.dismissed_datetime_kv_key, max(alert.datetime for alert in alerts)) return [] async def _produce_alerts_for_ipmitool_output(self, output): alerts = [] records = parse_ipmitool_output(output) if records: if await self.middleware.call("keyvalue.has_key", self.dismissed_datetime_kv_key): dismissed_datetime = (await self.middleware.call( "keyvalue.get", self.dismissed_datetime_kv_key)).replace(tzinfo=None) else: # Prevent notifying about existing alerts on first install/upgrade dismissed_datetime = max(record.datetime for record in records) await self.middleware.call("keyvalue.set", self.dismissed_datetime_kv_key, dismissed_datetime) for record in records: if record.datetime <= dismissed_datetime: continue title = "%(sensor)s %(direction)s %(event)s" if record.verbose is not None: title += ": %(verbose)s" args = dict(record._asdict()) args.pop("id") args.pop("datetime") alerts.append( Alert( title=title, args=args, datetime=record.datetime, )) return alerts
class ISCSIPortalIPAlertSource(AlertSource): schedule = IntervalSchedule(timedelta(minutes=60)) async def check(self): if not await self.middleware.call('service.started', 'iscsitarget'): return in_use_ips = { i['address'] for i in await self.middleware.call('interface.ip_in_use', {'any': True}) } portals = { p['id']: p for p in await self.middleware.call('iscsi.portal.query') } ips = [] for target in await self.middleware.call('iscsi.target.query'): for group in target['groups']: ips.extend( map( lambda ip: ip['ip'], filter(lambda a: a['ip'] not in in_use_ips, portals[group['portal']]['listen']))) if ips: return Alert(ISCSIPortalIPAlertClass, ', '.join(ips))
class HasUpdateAlertSource(ThreadedAlertSource): schedule = IntervalSchedule(timedelta(hours=1)) run_on_backup_node = False def check_sync(self): try: self.middleware.call_sync("datastore.query", "system.update", [], {"get": True}) except IndexError: self.middleware.call_sync("datastore.insert", "system.update", { "upd_autocheck": True, "upd_train": "", }) path = self.middleware.call_sync("update.get_update_location") if not path: return updates = None try: if PendingUpdates: updates = PendingUpdates(path) except Exception: pass if updates: return Alert(HasUpdateAlertClass)
class ZpoolCapacityAlertSource(AlertSource): schedule = IntervalSchedule(timedelta(minutes=5)) async def check(self): alerts = [] for pool in await self.middleware.call("zfs.pool.query"): try: capacity = int(pool["properties"]["capacity"]["parsed"]) except (KeyError, ValueError): continue for target_capacity, klass in [ (90, ZpoolCapacityCriticalAlertClass), (80, ZpoolCapacityWarningAlertClass), (70, ZpoolCapacityNoticeAlertClass), ]: if capacity >= target_capacity: alerts.append( Alert( klass, { "volume": pool["name"], "capacity": capacity, }, key=[pool["name"]], )) break elif capacity == target_capacity - 1: # If pool capacity is 89%, 79%, 69%, leave the alert in its previous state. # In other words, don't flap alert in case if pool capacity is oscilating around threshold value. raise UnavailableException() return alerts
class LDAPBindAlertSource(AlertSource): schedule = IntervalSchedule(timedelta(minutes=10)) run_on_backup_node = False async def check(self): if (await self.middleware.call('ldap.get_state')) == 'DISABLED': return try: await self.middleware.call("ldap.started") except Exception as e: return Alert(LDAPBindAlertClass, {'ldaperr': str(e)}, key=None)
class ZpoolCapacityAlertSource(ThreadedAlertSource): level = AlertLevel.WARNING title = "The capacity for the volume is above recommended value" schedule = IntervalSchedule(timedelta(minutes=5)) def check_sync(self): alerts = [] pools = [ pool["name"] for pool in self.middleware.call_sync("pool.query") ] + ["freenas-boot"] for pool in pools: proc = subprocess.Popen([ "zpool", "list", "-H", "-o", "cap", pool.encode("utf8"), ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf8") data = proc.communicate()[0] if proc.returncode != 0: continue try: cap = int(data.strip("\n").replace("%", "")) except ValueError: continue msg = ( "The capacity for the volume \"%(volume)s\" is currently at " "%(capacity)d%%, while the recommended value is below 80%%.") level = None if cap >= 90: level = AlertLevel.CRITICAL elif cap >= 80: level = AlertLevel.WARNING if level: alerts.append( Alert( msg, { "volume": pool, "capacity": cap, }, key=[pool, level.name], level=level, )) return alerts
class VolumeVersionAlertSource(AlertSource): schedule = IntervalSchedule(timedelta(minutes=5)) async def check(self): alerts = [] for pool in await self.middleware.call("pool.query"): if not await self.middleware.call("pool.is_upgraded", pool["id"]): alerts.append(Alert( VolumeVersionAlertClass, pool["name"], )) return alerts
class VolumeVersionAlertSource(ThreadedAlertSource): schedule = IntervalSchedule(timedelta(minutes=5)) def check_sync(self): alerts = [] for pool in self.middleware.call_sync("pool.query"): if not self.middleware.call_sync('pool.is_upgraded', pool["id"]): alerts.append(Alert( VolumeVersionAlertClass, pool["name"], )) return alerts
class ActiveDirectoryDomainBindAlertSource(AlertSource): schedule = IntervalSchedule(timedelta(minutes=10)) async def check(self): if (await self.middleware.call('activedirectory.get_state') ) == 'DISABLED': return try: await self.middleware.call("activedirectory.started") except Exception as e: return Alert(ActiveDirectoryDomainBindAlertClass, {'wberr': str(e)}, key=None)
class AlertSource: schedule = IntervalSchedule(timedelta()) run_on_backup_node = True def __init__(self, middleware): self.middleware = middleware @property def name(self): return self.__class__.__name__.replace("AlertSource", "") async def check(self): raise NotImplementedError
class IPMISELAlertSource(AlertSource): schedule = IntervalSchedule(timedelta(minutes=5)) dismissed_datetime_kv_key = "alert:ipmi_sel:dismissed_datetime" async def check(self): if not has_ipmi(): return return await self._produce_alerts_for_ipmitool_output( (await run(["ipmitool", "-c", "sel", "elist"], encoding="utf8")).stdout) async def _produce_alerts_for_ipmitool_output(self, output): alerts = [] records = parse_ipmitool_output(output) if records: if await self.middleware.call("keyvalue.has_key", self.dismissed_datetime_kv_key): dismissed_datetime = ((await self.middleware.call( "keyvalue.get", self.dismissed_datetime_kv_key)).replace(tzinfo=None)) else: # Prevent notifying about existing alerts on first install/upgrade dismissed_datetime = max(record.datetime for record in records) await self.middleware.call("keyvalue.set", self.dismissed_datetime_kv_key, dismissed_datetime) for record in records: if record.datetime <= dismissed_datetime: continue args = dict(record._asdict()) args.pop("id") args.pop("datetime") alerts.append( Alert( IPMISELAlertClass, args, key=[args, record.datetime.isoformat()], datetime=record.datetime, )) return alerts
class AlertSource: schedule = IntervalSchedule(timedelta()) products = ("CORE", "ENTERPRISE", "SCALE", "SCALE_ENTERPRISE") failover_related = False run_on_backup_node = True def __init__(self, middleware): self.middleware = middleware @property def name(self): return self.__class__.__name__.replace("AlertSource", "") async def check(self): raise NotImplementedError
class NISBindAlertSource(AlertSource): schedule = IntervalSchedule(timedelta(minutes=10)) run_on_backup_node = False async def check(self): if (await self.middleware.call('nis.get_state')) == 'DISABLED': return try: await self.middleware.call("nis.started") except Exception as e: await self.middleware.call('nis.set_state', DSStatus['FAULTED']) return Alert( NISBindAlertClass, {'niserr': str(e)}, key=None )
class ZpoolCapacityAlertSource(ThreadedAlertSource): schedule = IntervalSchedule(timedelta(minutes=5)) def check_sync(self): alerts = [] pools = [ pool["name"] for pool in self.middleware.call_sync("pool.query") ] + ["freenas-boot"] for pool in pools: proc = subprocess.Popen([ "zpool", "list", "-H", "-o", "cap", pool.encode("utf8"), ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf8") data = proc.communicate()[0] if proc.returncode != 0: continue try: cap = int(data.strip("\n").replace("%", "")) except ValueError: continue klass = None if cap >= 90: klass = ZpoolCapacityWarningAlertClass elif cap >= 80: klass = ZpoolCapacityCriticalAlertClass if klass: alerts.append( Alert( klass, { "volume": pool, "capacity": cap, }, key=[pool], )) return alerts
class UpdateNotAppliedAlertSource(ThreadedAlertSource): schedule = IntervalSchedule(timedelta(minutes=10)) def check_sync(self): if os.path.exists(UPDATE_APPLIED_SENTINEL): try: with open(UPDATE_APPLIED_SENTINEL, "rb") as f: data = json.loads(f.read().decode("utf8")) except Exception: log.error( "Could not load UPDATE APPLIED SENTINEL located at {0}". format(UPDATE_APPLIED_SENTINEL), exc_info=True) return update_applied, msg = is_update_applied(data["update_version"], create_alert=False) if update_applied: return Alert(UpdateNotAppliedAlertClass, msg)
class AlertSource: level = NotImplemented title = NotImplemented hardware = False onetime = False schedule = IntervalSchedule(timedelta()) run_on_passive_node = True def __init__(self, middleware): self.middleware = middleware @property def name(self): return self.__class__.__name__.replace("AlertSource", "") async def check(self): raise NotImplementedError
class IPMISELSpaceLeftAlertSource(AlertSource): schedule = IntervalSchedule(timedelta(minutes=5)) async def check(self): if not has_ipmi(): return return self._produce_alert_for_ipmitool_output( (await run(["ipmitool", "sel", "info"], encoding="utf8")).stdout) def _produce_alert_for_ipmitool_output(self, output): sel_information = parse_sel_information(output) if int(sel_information["Percent Used"].rstrip("%")) > 90: return Alert( IPMISELSpaceLeftAlertClass, { "free": sel_information["Free Space"], "used": sel_information["Percent Used"], }, key=None, )
class SMARTAlertSource(ThreadedAlertSource): level = AlertLevel.CRITICAL title = "SMART error" hardware = True schedule = IntervalSchedule(timedelta(minutes=5)) def check_sync(self): alerts = [] with SmartAlert() as sa: for msgs in sa.data.values(): if not msgs: continue for msg in msgs: if msg is None: continue alerts.append(Alert(msg)) return alerts
class NVDIMMAlertSource(ThreadedAlertSource): schedule = IntervalSchedule(timedelta(minutes=5)) products = ("ENTERPRISE", ) def check_sync(self): alerts = [] i = 0 while True: try: critical_health = sysctl.filter( f"dev.nvdimm.{i}.critical_health")[0].value nvdimm_health = sysctl.filter( f"dev.nvdimm.{i}.nvdimm_health")[0].value es_health = sysctl.filter(f"dev.nvdimm.{i}.es_health")[0].value except IndexError: return alerts else: alerts.extend( produce_nvdimm_alerts(i, critical_health, nvdimm_health, es_health)) i += 1
class VolumeVersionAlertSource(ThreadedAlertSource): schedule = IntervalSchedule(timedelta(minutes=5)) def check_sync(self): alerts = [] for pool in self.middleware.call_sync("pool.query"): if not self.middleware.call_sync('pool.is_upgraded', pool["id"]): alerts.append(Alert( VolumeVersionAlertClass, pool["name"], )) proc = subprocess.Popen( "zfs upgrade | grep FILESYSTEM", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf8", ) output = proc.communicate()[0].strip(" ").strip("\n") if output: alerts.append(Alert(ZfsVersionOutOfDateAlertClass)) return alerts
class QuotaAlertSource(ThreadedAlertSource): schedule = IntervalSchedule(timedelta(minutes=5)) def check_sync(self): alerts = [] datasets = self.middleware.call_sync( "zfs.dataset.query_for_quota_alert") pool_sizes = {} for d in datasets: d["name"] = d["name"]["rawvalue"] if "/" not in d["name"]: pool_sizes[d["name"]] = int(d["available"]["rawvalue"]) + int( d["used"]["rawvalue"]) for k, default in [("org.freenas:quota_warning", 80), ("org.freenas:quota_critical", 95), ("org.freenas:refquota_warning", 80), ("org.freenas:refquota_critical", 95)]: try: d[k] = int(d[k]["rawvalue"]) except (KeyError, ValueError): d[k] = default # call this outside the for loop since we don't need to check # for every dataset that could be potentially be out of quota... hostname = self.middleware.call_sync("system.hostname") datasets = sorted(datasets, key=lambda ds: ds["name"]) for dataset in datasets: for quota_property in ["quota", "refquota"]: try: quota_value = int(dataset[quota_property]["rawvalue"]) except (AttributeError, KeyError, ValueError): continue if quota_value == 0: continue if quota_property == "quota": # We can't use "used" property since it includes refreservation # But if "refquota" is smaller than "quota", then "available" will be reported with regards to # that smaller value, and we will get false positive try: refquota_value = int(dataset["refquota"]["rawvalue"]) except (AttributeError, KeyError, ValueError): continue else: if refquota_value and refquota_value < quota_value: continue # Quota larger than dataset available size will never be exceeded, # but will break out logic if quota_value > pool_sizes[dataset["name"].split("/")[0]]: continue used = quota_value - int(dataset["available"]["rawvalue"]) elif quota_property == "refquota": used = int(dataset["usedbydataset"]["rawvalue"]) else: raise RuntimeError() used_fraction = 100 * used / quota_value critical_threshold = dataset[ f"org.freenas:{quota_property}_critical"] warning_threshold = dataset[ f"org.freenas:{quota_property}_warning"] if critical_threshold != 0 and used_fraction >= critical_threshold: klass = QuotaCriticalAlertClass elif warning_threshold != 0 and used_fraction >= warning_threshold: klass = QuotaWarningAlertClass else: continue quota_name = quota_property[0].upper() + quota_property[1:] args = { "name": quota_name, "dataset": dataset["name"], "used_fraction": used_fraction, "used": format_size(used), "quota_value": format_size(quota_value), } mail = None owner = self._get_owner(dataset) if owner != 0: try: self.middleware.call_sync('user.get_user_obj', {'uid': owner}) user_exists = True except KeyError: user_exists = False to = None logger.debug("Unable to query bsduser with uid %r", owner) if user_exists: try: bsduser = self.middleware.call_sync( "datastore.query", "account.bsdusers", [["bsdusr_uid", "=", owner]], {"get": True}, ) to = bsduser["bsdusr_email"] or None except IndexError: to = None if to is not None: mail = { "to": [to], "subject": f"{hostname}: {quota_name} exceeded on dataset {dataset['name']}", "text": klass.text % args } alerts.append( Alert( klass, args=args, key=[dataset["name"], quota_property], mail=mail, )) return alerts def _get_owner(self, dataset): mountpoint = None if dataset["mounted"]["value"] == "yes": if dataset["mountpoint"]["value"] == "legacy": for m in (getmntinfo() if getmntinfo else []): if m.source == dataset["name"]: mountpoint = m.dest break else: mountpoint = dataset["mountpoint"]["value"] if mountpoint is None: logger.debug( "Unable to get mountpoint for dataset %r, assuming owner = root", dataset["name"]) uid = 0 else: try: stat_info = os.stat(mountpoint) except Exception: logger.debug( "Unable to stat mountpoint %r, assuming owner = root", mountpoint) uid = 0 else: uid = stat_info.st_uid return uid
class IPMISELAlertSource(AlertSource): schedule = IntervalSchedule(timedelta(minutes=5)) dismissed_datetime_kv_key = "alert:ipmi_sel:dismissed_datetime" # https://github.com/openbmc/ipmitool/blob/master/include/ipmitool/ipmi_sel.h#L297 IPMI_SENSORS = ( "Redundancy State", "Temperature", "Voltage", "Current", "Fan", "Physical Security", "Platform Security", "Processor", "Power Supply", "Memory", "System Firmware Error", "Critical Interrupt", "Management Subsystem Health", "Battery", ) IPMI_EVENTS_WHITELIST = ( ("Power Unit", "Soft-power control failure"), ("Power Unit", "Failure detected"), ("Power Unit", "Predictive failure"), ("Event Logging Disabled", "Log full"), ("Event Logging Disabled", "Log almost full"), ("System Event", "Undetermined system hardware failure"), ("Cable/Interconnect", "Config Error"), ) IPMI_EVENTS_BLACKLIST = ( ("Redundancy State", "Fully Redundant"), ("Processor", "Presence detected"), ("Power Supply", "Presence detected"), ) async def check(self): if not has_ipmi(): return return await self._produce_alerts_for_ipmitool_output(await ipmitool( ["-c", "sel", "elist"])) async def _produce_alerts_for_ipmitool_output(self, output): alerts = [] records = parse_ipmitool_output(output) records = [ record for record in records if ((any( record.sensor.startswith(f"{sensor} #0x") for sensor in self.IPMI_SENSORS) or any( record.sensor.startswith(f"{sensor} #0x") and record.event == event for sensor, event in self.IPMI_EVENTS_WHITELIST)) and not any( record.sensor.startswith(f"{sensor} #0x") and record.event == event for sensor, event in self.IPMI_EVENTS_BLACKLIST)) ] if records: if await self.middleware.call("keyvalue.has_key", self.dismissed_datetime_kv_key): dismissed_datetime = ((await self.middleware.call( "keyvalue.get", self.dismissed_datetime_kv_key)).replace(tzinfo=None)) else: # Prevent notifying about existing alerts on first install/upgrade dismissed_datetime = max(record.datetime for record in records) await self.middleware.call("keyvalue.set", self.dismissed_datetime_kv_key, dismissed_datetime) for record in records: if record.datetime <= dismissed_datetime: continue args = dict(record._asdict()) args.pop("id") args.pop("datetime") alerts.append( Alert( IPMISELAlertClass, args, key=[args, record.datetime.isoformat()], datetime=record.datetime, )) return alerts