def update_summary(self):
def update_dict(d1, d2):
for k in d2:
if k in d1:
d1[k] += d2[k]
else:
d1[k] = d2[k]
services = SummaryItem.items_to_dict(self.direct_services)
subscribers = SummaryItem.items_to_dict(self.direct_subscribers)
objects = {
self.managed_object.object_profile.id: 1
}
for a in ActiveAlarm.objects.filter(root=self.id):
a.update_summary()
update_dict(
objects,
SummaryItem.items_to_dict(a.total_objects)
)
update_dict(
services,
SummaryItem.items_to_dict(a.total_services)
)
update_dict(
subscribers,
SummaryItem.items_to_dict(a.total_subscribers)
)
obj_list = ObjectSummaryItem.dict_to_items(objects)
svc_list = SummaryItem.dict_to_items(services)
sub_list = SummaryItem.dict_to_items(subscribers)
if svc_list != self.total_services or sub_list != self.total_subscribers or obj_list != self.total_objects:
ns = ServiceSummary.get_severity({
"service": services,
"subscriber": subscribers,
"objects": objects
})
self.total_objects = obj_list
self.total_services = svc_list
self.total_subscribers = sub_list
if ns != self.severity:
self.change_severity(severity=ns, to_save=False)
self.safe_save()
def raise_alarm(self, r, e):
managed_object = self.eval_expression(r.managed_object, event=e)
if not managed_object:
self.logger.info("Empty managed object, ignoring")
return
# @todo: Make configurable
if not managed_object.is_managed:
self.logger.info(
"Managed object is not managed. Do not raise alarm")
return
if e.managed_object.id != managed_object.id:
metrics["alarm_change_mo"] += 1
self.logger.info("Changing managed object to %s",
managed_object.name)
discriminator, vars = r.get_vars(e)
if r.unique:
assert discriminator is not None
a = ActiveAlarm.objects.filter(
managed_object=managed_object.id,
discriminator=discriminator).first()
if not a:
# Try to reopen alarm
a = ArchivedAlarm.objects.filter(
managed_object=managed_object.id,
discriminator=discriminator,
control_time__gte=e.timestamp).first()
if a:
# Reopen alarm
self.logger.info("[%s|%s|%s] %s reopens alarm %s(%s)",
e.id, managed_object.name,
managed_object.address,
e.event_class.name, a.alarm_class.name,
a.id)
a = a.reopen("Reopened by disposition rule '%s'" %
r.u_name)
metrics["alarm_reopen"] += 1
if a:
# Active alarm found, refresh
self.logger.info(
"[%s|%s|%s] Contributing event %s to active alarm %s(%s)",
e.id, managed_object.name, managed_object.address,
e.event_class.name, a.alarm_class.name, a.id)
# Contribute event to alarm
e.contribute_to_alarm(a)
if e.timestamp < a.timestamp:
# Set to earlier date
a.timestamp = e.timestamp
a.save()
elif e.timestamp > a.last_update:
# Refresh last update
a.last_update = e.timestamp
a.save()
metrics["alarm_contribute"] += 1
return
# Calculate alarm coverage
summary = ServiceSummary.get_object_summary(managed_object)
summary["object"] = {managed_object.object_profile.id: 1}
#
severity = max(ServiceSummary.get_severity(summary), 1)
self.logger.info("[%s|%s|%s] %s: Calculated alarm severity is: %s",
e.id, managed_object.name, managed_object.address,
r.u_name, severity)
# Create new alarm
a = ActiveAlarm(
timestamp=e.timestamp,
last_update=e.timestamp,
managed_object=managed_object.id,
alarm_class=r.alarm_class,
severity=severity,
vars=vars,
discriminator=discriminator,
direct_services=SummaryItem.dict_to_items(summary["service"]),
direct_subscribers=SummaryItem.dict_to_items(
summary["subscriber"]),
total_objects=ObjectSummaryItem.dict_to_items(summary["object"]),
total_services=SummaryItem.dict_to_items(summary["service"]),
total_subscribers=SummaryItem.dict_to_items(summary["subscriber"]),
log=[
AlarmLog(timestamp=datetime.datetime.now(),
from_status="A",
to_status="A",
message="Alarm risen from event %s(%s) by rule '%s'" %
(str(e.id), str(e.event_class.name), r.u_name))
],
opening_event=e.id)
a.save()
e.contribute_to_alarm(a)
self.logger.info("[%s|%s|%s] %s raises alarm %s(%s): %r", e.id,
managed_object.name, managed_object.address,
e.event_class.name, a.alarm_class.name, a.id, a.vars)
metrics["alarm_raise"] += 1
self.correlate(r, a)
# Notify about new alarm
if not a.root:
a.managed_object.event(
a.managed_object.EV_ALARM_RISEN, {
"alarm": a,
"subject": a.subject,
"body": a.body,
"symptoms": a.alarm_class.symptoms,
"recommended_actions": a.alarm_class.recommended_actions,
"probable_causes": a.alarm_class.probable_causes
},
delay=a.alarm_class.get_notification_delay())
# Gather diagnostics when necessary
AlarmDiagnosticConfig.on_raise(a)
# Watch for escalations, when necessary
if config.correlator.auto_escalation and not a.root:
AlarmEscalation.watch_escalations(a)
def _get_path_summary_bulk(self):
def list_to_dict(summary):
if not summary:
return {}
return {d["profile"]: d["summary"] for d in summary}
def e_list_to_dict(summary):
if not summary:
return {}
return {d.profile: d.summary for d in summary}
def dict_to_list(d):
return [{"profile": k, "summary": d[k]} for k in sorted(d)]
def get_summary(docs, key, direct=None):
r = direct.copy() if direct else {}
for doc in docs:
dv = doc.get(key)
if not dv:
continue
for k in dv:
nv = dv[k]
if nv:
r[k] = r.get(k, 0) + nv
return r
def get_root_path(alarm_id, path=None):
path = path or []
if alarm_id in path:
raise ValueError("Loop detected: %s" % (str(x) for x in path))
path = path + [alarm_id]
root = alarms[alarm_id].get("root")
if not root:
return path
return get_root_path(root, path)
alarms = {} # id -> alarm doc
children = defaultdict(list) # id -> [alarm doc, ..]
# Inject current alarm
alarms[self.id] = {
"_id": self.id,
"root": self.root,
"severity": self.severity,
"total_objects": e_list_to_dict(self.total_objects),
"total_services": e_list_to_dict(self.total_services),
"total_subscribers": e_list_to_dict(self.total_subscribers),
}
# Collect relevant neighbors
for doc in ActiveAlarm._get_collection().aggregate([
# Starting from given alarm
{
"$match": {
"_id": self.root
}
},
# Add to 'path' field all alarm upwards
{
"$graphLookup": {
"from": "noc.alarms.active",
"connectFromField": "root",
"connectToField": "_id",
"startWith": "$root",
"as": "path",
"maxDepth": 50,
}
},
# Append the necessary fields of given alarm to 'path' field
# and wipe out all other fields
{
"$project": {
"_id": 0,
"path": {
"$concatArrays": [
"$path",
[{
"_id": "$_id",
"root": "$root",
"severity": "$severity",
"direct_services": "$direct_services",
"direct_subscribers": "$direct_subscribers",
"total_objects": "$total_objects",
"total_services": "$total_services",
"total_subscribers": "$total_subscribers",
}],
]
},
}
},
# Convert path field to the list of documents
{
"$unwind": "$path"
},
# Normalize resulting documents
{
"$project": {
"_id": "$path._id",
"root": "$path.root",
"severity": "$path.severity",
"direct_services": "$path.direct_services",
"direct_subscribers": "$path.direct_subscribers",
"total_objects": "$path.total_objects",
"total_services": "$path.total_services",
"total_subscribers": "$path.total_subscribers",
}
},
# Add all children alarms to 'children' field
{
"$lookup": {
"from": "noc.alarms.active",
"localField": "_id",
"foreignField": "root",
"as": "children",
}
},
# Append the neccessary fields of path alarms to `children` field
# and wipe out all other fields
{
"$project": {
"_id": 0,
"children": {
"$concatArrays": [
"$children",
[{
"_id": "$_id",
"root": "$root",
"severity": "$severity",
"direct_services": "$direct_services",
"direct_subscribers": "$direct_subscribers",
"total_objects": "$total_objects",
"total_services": "$total_services",
"total_subscribers": "$total_subscribers",
}],
]
},
}
},
# Convert path field to the list of documents
{
"$unwind": "$children"
},
# Normalize resulting documents
{
"$project": {
"_id": "$children._id",
"root": "$children.root",
"severity": "$children.severity",
"direct_services": "$children.direct_services",
"direct_subscribers": "$children.direct_subscribers",
"total_objects": "$children.total_objects",
"total_services": "$children.total_services",
"total_subscribers": "$children.total_subscribers",
}
},
]):
# May contains duplicates, perform deduplication
doc["direct_services"] = list_to_dict(doc.get("direct_services"))
doc["direct_subscribers"] = list_to_dict(
doc.get("direct_subscribers"))
doc["total_objects"] = list_to_dict(doc.get("total_objects"))
doc["total_services"] = list_to_dict(doc.get("total_services"))
doc["total_subscribers"] = list_to_dict(
doc.get("total_subscribers"))
if doc["_id"] == self.id:
doc["root"] = self.root
alarms[doc["_id"]] = doc
for doc in alarms.values():
children[doc.get("root")] += [doc]
# Get path to from current root upwards to global root
# Check for loops, raise Value error if loop detected
root_path = get_root_path(self.root)
bulk = []
now = datetime.datetime.now()
for root in root_path:
doc = alarms[root]
consequences = children[root]
total_objects = get_summary(
consequences, "total_objects",
{self.managed_object.object_profile.id: 1})
total_services = get_summary(consequences, "total_services",
doc.get("direct_services"))
total_subscribers = get_summary(consequences, "total_subscribers",
doc.get("direct_subscribers"))
if (doc["total_objects"] != total_objects
or doc["total_services"] != total_services
or doc["total_subscribers"] != total_subscribers):
# Changed
severity = ServiceSummary.get_severity({
"service":
total_services,
"subscriber":
total_subscribers,
"objects":
total_objects,
})
op = {
"$set": {
"severity": severity,
"total_objects": dict_to_list(total_objects),
"total_services": dict_to_list(total_services),
"total_subscribers": dict_to_list(total_subscribers),
}
}
if severity != doc.get("severity"):
op["$push"] = {
"log": {
"timestamp": now,
"from_status": "A",
"to_status": "A",
"message": "Severity changed to %d" % severity,
}
}
bulk += [UpdateOne({"_id": root}, op)]
return bulk
