def update_summary(self): def update_dict(d1, d2): for k in d2: if k in d1: d1[k] += d2[k] else: d1[k] = d2[k] services = SummaryItem.items_to_dict(self.direct_services) subscribers = SummaryItem.items_to_dict(self.direct_subscribers) objects = { self.managed_object.object_profile.id: 1 } for a in ActiveAlarm.objects.filter(root=self.id): a.update_summary() update_dict( objects, SummaryItem.items_to_dict(a.total_objects) ) update_dict( services, SummaryItem.items_to_dict(a.total_services) ) update_dict( subscribers, SummaryItem.items_to_dict(a.total_subscribers) ) obj_list = ObjectSummaryItem.dict_to_items(objects) svc_list = SummaryItem.dict_to_items(services) sub_list = SummaryItem.dict_to_items(subscribers) if svc_list != self.total_services or sub_list != self.total_subscribers or obj_list != self.total_objects: ns = ServiceSummary.get_severity({ "service": services, "subscriber": subscribers, "objects": objects }) self.total_objects = obj_list self.total_services = svc_list self.total_subscribers = sub_list if ns != self.severity: self.change_severity(severity=ns, to_save=False) self.safe_save()
def raise_alarm(self, r, e): managed_object = self.eval_expression(r.managed_object, event=e) if not managed_object: self.logger.info("Empty managed object, ignoring") return # @todo: Make configurable if not managed_object.is_managed: self.logger.info( "Managed object is not managed. Do not raise alarm") return if e.managed_object.id != managed_object.id: metrics["alarm_change_mo"] += 1 self.logger.info("Changing managed object to %s", managed_object.name) discriminator, vars = r.get_vars(e) if r.unique: assert discriminator is not None a = ActiveAlarm.objects.filter( managed_object=managed_object.id, discriminator=discriminator).first() if not a: # Try to reopen alarm a = ArchivedAlarm.objects.filter( managed_object=managed_object.id, discriminator=discriminator, control_time__gte=e.timestamp).first() if a: # Reopen alarm self.logger.info("[%s|%s|%s] %s reopens alarm %s(%s)", e.id, managed_object.name, managed_object.address, e.event_class.name, a.alarm_class.name, a.id) a = a.reopen("Reopened by disposition rule '%s'" % r.u_name) metrics["alarm_reopen"] += 1 if a: # Active alarm found, refresh self.logger.info( "[%s|%s|%s] Contributing event %s to active alarm %s(%s)", e.id, managed_object.name, managed_object.address, e.event_class.name, a.alarm_class.name, a.id) # Contribute event to alarm e.contribute_to_alarm(a) if e.timestamp < a.timestamp: # Set to earlier date a.timestamp = e.timestamp a.save() elif e.timestamp > a.last_update: # Refresh last update a.last_update = e.timestamp a.save() metrics["alarm_contribute"] += 1 return # Calculate alarm coverage summary = ServiceSummary.get_object_summary(managed_object) summary["object"] = {managed_object.object_profile.id: 1} # severity = max(ServiceSummary.get_severity(summary), 1) self.logger.info("[%s|%s|%s] %s: Calculated alarm severity is: %s", e.id, managed_object.name, managed_object.address, r.u_name, severity) # Create new alarm a = ActiveAlarm( timestamp=e.timestamp, last_update=e.timestamp, managed_object=managed_object.id, alarm_class=r.alarm_class, severity=severity, vars=vars, discriminator=discriminator, direct_services=SummaryItem.dict_to_items(summary["service"]), direct_subscribers=SummaryItem.dict_to_items( summary["subscriber"]), total_objects=ObjectSummaryItem.dict_to_items(summary["object"]), total_services=SummaryItem.dict_to_items(summary["service"]), total_subscribers=SummaryItem.dict_to_items(summary["subscriber"]), log=[ AlarmLog(timestamp=datetime.datetime.now(), from_status="A", to_status="A", message="Alarm risen from event %s(%s) by rule '%s'" % (str(e.id), str(e.event_class.name), r.u_name)) ], opening_event=e.id) a.save() e.contribute_to_alarm(a) self.logger.info("[%s|%s|%s] %s raises alarm %s(%s): %r", e.id, managed_object.name, managed_object.address, e.event_class.name, a.alarm_class.name, a.id, a.vars) metrics["alarm_raise"] += 1 self.correlate(r, a) # Notify about new alarm if not a.root: a.managed_object.event( a.managed_object.EV_ALARM_RISEN, { "alarm": a, "subject": a.subject, "body": a.body, "symptoms": a.alarm_class.symptoms, "recommended_actions": a.alarm_class.recommended_actions, "probable_causes": a.alarm_class.probable_causes }, delay=a.alarm_class.get_notification_delay()) # Gather diagnostics when necessary AlarmDiagnosticConfig.on_raise(a) # Watch for escalations, when necessary if config.correlator.auto_escalation and not a.root: AlarmEscalation.watch_escalations(a)
def _get_path_summary_bulk(self): def list_to_dict(summary): if not summary: return {} return {d["profile"]: d["summary"] for d in summary} def e_list_to_dict(summary): if not summary: return {} return {d.profile: d.summary for d in summary} def dict_to_list(d): return [{"profile": k, "summary": d[k]} for k in sorted(d)] def get_summary(docs, key, direct=None): r = direct.copy() if direct else {} for doc in docs: dv = doc.get(key) if not dv: continue for k in dv: nv = dv[k] if nv: r[k] = r.get(k, 0) + nv return r def get_root_path(alarm_id, path=None): path = path or [] if alarm_id in path: raise ValueError("Loop detected: %s" % (str(x) for x in path)) path = path + [alarm_id] root = alarms[alarm_id].get("root") if not root: return path return get_root_path(root, path) alarms = {} # id -> alarm doc children = defaultdict(list) # id -> [alarm doc, ..] # Inject current alarm alarms[self.id] = { "_id": self.id, "root": self.root, "severity": self.severity, "total_objects": e_list_to_dict(self.total_objects), "total_services": e_list_to_dict(self.total_services), "total_subscribers": e_list_to_dict(self.total_subscribers), } # Collect relevant neighbors for doc in ActiveAlarm._get_collection().aggregate([ # Starting from given alarm { "$match": { "_id": self.root } }, # Add to 'path' field all alarm upwards { "$graphLookup": { "from": "noc.alarms.active", "connectFromField": "root", "connectToField": "_id", "startWith": "$root", "as": "path", "maxDepth": 50, } }, # Append the necessary fields of given alarm to 'path' field # and wipe out all other fields { "$project": { "_id": 0, "path": { "$concatArrays": [ "$path", [{ "_id": "$_id", "root": "$root", "severity": "$severity", "direct_services": "$direct_services", "direct_subscribers": "$direct_subscribers", "total_objects": "$total_objects", "total_services": "$total_services", "total_subscribers": "$total_subscribers", }], ] }, } }, # Convert path field to the list of documents { "$unwind": "$path" }, # Normalize resulting documents { "$project": { "_id": "$path._id", "root": "$path.root", "severity": "$path.severity", "direct_services": "$path.direct_services", "direct_subscribers": "$path.direct_subscribers", "total_objects": "$path.total_objects", "total_services": "$path.total_services", "total_subscribers": "$path.total_subscribers", } }, # Add all children alarms to 'children' field { "$lookup": { "from": "noc.alarms.active", "localField": "_id", "foreignField": "root", "as": "children", } }, # Append the neccessary fields of path alarms to `children` field # and wipe out all other fields { "$project": { "_id": 0, "children": { "$concatArrays": [ "$children", [{ "_id": "$_id", "root": "$root", "severity": "$severity", "direct_services": "$direct_services", "direct_subscribers": "$direct_subscribers", "total_objects": "$total_objects", "total_services": "$total_services", "total_subscribers": "$total_subscribers", }], ] }, } }, # Convert path field to the list of documents { "$unwind": "$children" }, # Normalize resulting documents { "$project": { "_id": "$children._id", "root": "$children.root", "severity": "$children.severity", "direct_services": "$children.direct_services", "direct_subscribers": "$children.direct_subscribers", "total_objects": "$children.total_objects", "total_services": "$children.total_services", "total_subscribers": "$children.total_subscribers", } }, ]): # May contains duplicates, perform deduplication doc["direct_services"] = list_to_dict(doc.get("direct_services")) doc["direct_subscribers"] = list_to_dict( doc.get("direct_subscribers")) doc["total_objects"] = list_to_dict(doc.get("total_objects")) doc["total_services"] = list_to_dict(doc.get("total_services")) doc["total_subscribers"] = list_to_dict( doc.get("total_subscribers")) if doc["_id"] == self.id: doc["root"] = self.root alarms[doc["_id"]] = doc for doc in alarms.values(): children[doc.get("root")] += [doc] # Get path to from current root upwards to global root # Check for loops, raise Value error if loop detected root_path = get_root_path(self.root) bulk = [] now = datetime.datetime.now() for root in root_path: doc = alarms[root] consequences = children[root] total_objects = get_summary( consequences, "total_objects", {self.managed_object.object_profile.id: 1}) total_services = get_summary(consequences, "total_services", doc.get("direct_services")) total_subscribers = get_summary(consequences, "total_subscribers", doc.get("direct_subscribers")) if (doc["total_objects"] != total_objects or doc["total_services"] != total_services or doc["total_subscribers"] != total_subscribers): # Changed severity = ServiceSummary.get_severity({ "service": total_services, "subscriber": total_subscribers, "objects": total_objects, }) op = { "$set": { "severity": severity, "total_objects": dict_to_list(total_objects), "total_services": dict_to_list(total_services), "total_subscribers": dict_to_list(total_subscribers), } } if severity != doc.get("severity"): op["$push"] = { "log": { "timestamp": now, "from_status": "A", "to_status": "A", "message": "Severity changed to %d" % severity, } } bulk += [UpdateOne({"_id": root}, op)] return bulk