def raise_alarm(self, r, e): managed_object = self.eval_expression(r.managed_object, event=e) if not managed_object: self.logger.info("Empty managed object, ignoring") return # @todo: Make configurable if not managed_object.is_managed: self.logger.info( "Managed object is not managed. Do not raise alarm") return if e.managed_object.id != managed_object.id: metrics["alarm_change_mo"] += 1 self.logger.info("Changing managed object to %s", managed_object.name) discriminator, vars = r.get_vars(e) if r.unique: assert discriminator is not None a = ActiveAlarm.objects.filter( managed_object=managed_object.id, discriminator=discriminator).first() if not a: # Try to reopen alarm a = ArchivedAlarm.objects.filter( managed_object=managed_object.id, discriminator=discriminator, control_time__gte=e.timestamp).first() if a: # Reopen alarm self.logger.info("[%s|%s|%s] %s reopens alarm %s(%s)", e.id, managed_object.name, managed_object.address, e.event_class.name, a.alarm_class.name, a.id) a = a.reopen("Reopened by disposition rule '%s'" % r.u_name) metrics["alarm_reopen"] += 1 if a: # Active alarm found, refresh self.logger.info( "[%s|%s|%s] Contributing event %s to active alarm %s(%s)", e.id, managed_object.name, managed_object.address, e.event_class.name, a.alarm_class.name, a.id) # Contribute event to alarm e.contribute_to_alarm(a) if e.timestamp < a.timestamp: # Set to earlier date a.timestamp = e.timestamp a.save() elif e.timestamp > a.last_update: # Refresh last update a.last_update = e.timestamp a.save() metrics["alarm_contribute"] += 1 return # Calculate alarm coverage summary = ServiceSummary.get_object_summary(managed_object) summary["object"] = {managed_object.object_profile.id: 1} # severity = max(ServiceSummary.get_severity(summary), 1) self.logger.info("[%s|%s|%s] %s: Calculated alarm severity is: %s", e.id, managed_object.name, managed_object.address, r.u_name, severity) # Create new alarm a = ActiveAlarm( timestamp=e.timestamp, last_update=e.timestamp, managed_object=managed_object.id, alarm_class=r.alarm_class, severity=severity, vars=vars, discriminator=discriminator, direct_services=SummaryItem.dict_to_items(summary["service"]), direct_subscribers=SummaryItem.dict_to_items( summary["subscriber"]), total_objects=ObjectSummaryItem.dict_to_items(summary["object"]), total_services=SummaryItem.dict_to_items(summary["service"]), total_subscribers=SummaryItem.dict_to_items(summary["subscriber"]), log=[ AlarmLog(timestamp=datetime.datetime.now(), from_status="A", to_status="A", message="Alarm risen from event %s(%s) by rule '%s'" % (str(e.id), str(e.event_class.name), r.u_name)) ], opening_event=e.id) a.save() e.contribute_to_alarm(a) self.logger.info("[%s|%s|%s] %s raises alarm %s(%s): %r", e.id, managed_object.name, managed_object.address, e.event_class.name, a.alarm_class.name, a.id, a.vars) metrics["alarm_raise"] += 1 self.correlate(r, a) # Notify about new alarm if not a.root: a.managed_object.event( a.managed_object.EV_ALARM_RISEN, { "alarm": a, "subject": a.subject, "body": a.body, "symptoms": a.alarm_class.symptoms, "recommended_actions": a.alarm_class.recommended_actions, "probable_causes": a.alarm_class.probable_causes }, delay=a.alarm_class.get_notification_delay()) # Gather diagnostics when necessary AlarmDiagnosticConfig.on_raise(a) # Watch for escalations, when necessary if config.correlator.auto_escalation and not a.root: AlarmEscalation.watch_escalations(a)
def update_umbrella(self, umbrella_cls, details): """ Update umbrella alarm status for managed object :param umbrella_cls: Umbrella alarm class (AlarmClass instance) :param details: List of dicts, containing * alarm_class - Detail alarm class * path - Additional path * severity - Alarm severity * vars - dict of alarm vars :return: """ from noc.fm.models.activealarm import ActiveAlarm now = datetime.datetime.now() umbrella = ActiveAlarm.objects.filter( alarm_class=umbrella_cls.id, managed_object=self.object.id).first() u_sev = sum(d.get("severity", 0) for d in details) if not umbrella and not details: # No money, no honey return elif not umbrella and details: # Open new umbrella umbrella = ActiveAlarm(timestamp=now, managed_object=self.object.id, alarm_class=umbrella_cls.id, severity=u_sev) umbrella.save() self.logger.info("Opening umbrella alarm %s (%s)", umbrella.id, umbrella_cls.name) elif umbrella and not details: # Close existing umbrella self.logger.info("Clearing umbrella alarm %s (%s)", umbrella.id, umbrella_cls.name) umbrella.clear_alarm("Closing umbrella") elif umbrella and details and u_sev != umbrella.severity: self.logger.info("Change umbrella alarm %s severity %s -> %s (%s)", umbrella.id, umbrella.severity, u_sev, umbrella_cls.name) umbrella.change_severity(severity=u_sev) # Get existing details for umbrella active_details = {} # (alarm class, path) -> alarm if umbrella: for da in ActiveAlarm.objects.filter(root=umbrella.id): d_path = da.vars.get("path", "") active_details[da.alarm_class, d_path] = da # Synchronize details self.logger.info("Active details: %s" % active_details) seen = set() for d in details: d_path = d.get("path", "") d_key = (d["alarm_class"], d_path) d_sev = d.get("severity", 0) # Key for seen details seen.add(d_key) if d_key in active_details and active_details[ d_key].severity != d_sev: # Change severity self.logger.info("Change detail alarm %s severity %s -> %s", active_details[d_key].id, active_details[d_key].severity, d_sev) active_details[d_key].change_severity(severity=d_sev) elif d_key not in active_details: # Create alarm self.logger.info("Create detail alarm to path %s", d_key) v = d.get("vars", {}) v["path"] = d_path da = ActiveAlarm(timestamp=now, managed_object=self.object.id, alarm_class=d["alarm_class"], severity=d_sev, vars=v, root=umbrella.id) da.save() self.logger.info("Opening detail alarm %s %s (%s)", da.id, d_path, da.alarm_class.name) # Close details when necessary for d in set(active_details) - seen: self.logger.info("Clearing detail alarm %s", active_details[d].id) active_details[d].clear_alarm("Closing")