class NewEvent(document.Document):
"""
Raw unclassified event as written by SAE or collector
"""
meta = {
"collection": "noc.events.new",
"allow_inheritance": False,
"indexes": ["seq"]
}
status = "N"
# Fields
timestamp = fields.DateTimeField(required=True)
managed_object = nosql.ForeignKeyField(ManagedObject, required=True)
raw_vars = nosql.RawDictField(required=True)
log = fields.ListField(fields.EmbeddedDocumentField(EventLog))
seq = fields.BinaryField(max_bytes=8, required=False)
def __unicode__(self):
return unicode(self.id)
def mark_as_failed(self, version, traceback):
"""
Move event into noc.events.failed
"""
message = "Failed to classify on NOC version %s" % version
log = self.log + [
EventLog(timestamp=datetime.datetime.now(),
from_status="N",
to_status="F",
message=message)
]
e = FailedEvent(id=self.id,
timestamp=self.timestamp,
managed_object=self.managed_object,
raw_vars=self.raw_vars,
version=version,
traceback=traceback,
log=log)
e.save()
self.delete()
return e
@property
def source(self):
"""
Event source or None
"""
if self.raw_vars and "source" in self.raw_vars:
return self.raw_vars["source"]
return None
def log_message(self, message):
self.log += [
EventLog(timestamp=datetime.datetime.now(),
from_status=self.status,
to_status=self.status,
message=message)
]
self.save()
class ArchivedEvent(document.Document):
"""
"""
meta = {
"collection": "noc.events.archive",
"strict": False,
"auto_create_index": False,
"indexes": ["timestamp", "alarms"],
}
status = "S"
timestamp = fields.DateTimeField(required=True)
managed_object = nosql.ForeignKeyField(ManagedObject, required=True)
event_class = nosql.PlainReferenceField(EventClass, required=True)
start_timestamp = fields.DateTimeField(required=True)
repeats = fields.IntField(required=True)
raw_vars = nosql.RawDictField()
resolved_vars = nosql.RawDictField()
vars = fields.DictField()
log = fields.ListField(nosql.EmbeddedDocumentField(EventLog))
alarms = fields.ListField(nosql.ObjectIdField())
def __str__(self):
return "%s" % self.id
@property
def duration(self):
"""
Logged event duration in seconds
"""
return (self.timestamp - self.start_timestamp).total_seconds()
def get_template_vars(self):
"""
Prepare template variables
"""
vars = self.vars.copy()
vars.update({"event": self})
return vars
@property
def subject(self):
ctx = Context(self.get_template_vars())
s = Template(self.event_class.subject_template).render(ctx)
if len(s) >= 255:
s = s[:125] + " ... " + s[-125:]
return s
@property
def body(self):
ctx = Context(self.get_template_vars())
s = Template(self.event_class.body_template).render(ctx)
return s
class FailedEvent(document.Document):
"""
Events that caused noc-classifier traceback
"""
meta = {"collection": "noc.events.failed", "allow_inheritance": False}
status = "F"
# Fields
timestamp = fields.DateTimeField(required=True)
managed_object = nosql.ForeignKeyField(ManagedObject, required=True)
raw_vars = nosql.RawDictField(required=True)
# NOC version caused traceback
version = fields.StringField(required=True)
traceback = fields.StringField()
log = fields.ListField(fields.EmbeddedDocumentField(EventLog))
def __unicode__(self):
return unicode(self.id)
def mark_as_new(self, message=None):
"""
Move to unclassified queue
"""
if message is None:
message = "Reclassification requested"
log = self.log + [
EventLog(timestamp=datetime.datetime.now(),
from_status="F",
to_status="N",
message=message)
]
e = NewEvent(id=self.id,
timestamp=self.timestamp,
managed_object=self.managed_object,
raw_vars=self.raw_vars,
log=log)
e.save()
self.delete()
return e
def log_message(self, message):
self.log += [
EventLog(timestamp=datetime.datetime.now(),
from_status=self.status,
to_status=self.status,
message=message)
]
self.save()
class ActiveAlarm(nosql.Document):
meta = {
"collection":
"noc.alarms.active",
"allow_inheritance":
False,
"indexes": [
"timestamp", "discriminator", "root", "-severity", "alarm_class",
("timestamp", "managed_object")
]
}
status = "A"
timestamp = nosql.DateTimeField(required=True)
last_update = nosql.DateTimeField(required=True)
managed_object = nosql.ForeignKeyField(ManagedObject)
alarm_class = nosql.PlainReferenceField(AlarmClass)
severity = nosql.IntField(required=True)
vars = nosql.DictField()
# Calculated alarm discriminator
# Has meaning only for alarms with is_unique flag set
# Calculated as sha1("value1\x00....\x00valueN").hexdigest()
discriminator = nosql.StringField(required=False)
log = nosql.ListField(nosql.EmbeddedDocumentField(AlarmLog))
# Responsible person
owner = nosql.ForeignKeyField(User, required=False)
#
opening_event = nosql.ObjectIdField(required=False)
closing_event = nosql.ObjectIdField(required=False)
# List of subscribers
subscribers = nosql.ListField(nosql.ForeignKeyField(User))
#
custom_subject = nosql.StringField(required=False)
custom_style = nosql.ForeignKeyField(Style, required=False)
#
reopens = nosql.IntField(required=False)
# RCA
# Reference to root cause (Active Alarm or Archived Alarm instance)
root = nosql.ObjectIdField(required=False)
def __unicode__(self):
return u"%s" % self.id
def save(self, *args, **kwargs):
if not self.last_update:
self.last_update = self.timestamp
return super(ActiveAlarm, self).save(*args, **kwargs)
def _change_root_severity(self):
"""
Change root severity, when necessary
"""
if not self.root:
return
root = get_alarm(self.root)
if root and root.severity < self.severity:
root.change_severity(self.severity)
root.log_message("Severity has been increased by child alarm %s" %
self.id)
def change_severity(self, user="", delta=None, severity=None):
"""
Change alarm severity
"""
if isinstance(user, User):
user = user.username
if delta:
self.severity = max(0, self.severity + delta)
if delta > 0:
self.log_message("%s has increased alarm severity by %s" %
(user, delta))
else:
self.log_message("%s has decreased alarm severity by %s" %
(user, delta))
elif severity:
self.severity = severity.severity
self.log_message("%s has changed severity to %s" %
(user, severity.name))
self._change_root_severity()
self.save()
def log_message(self, message, to_save=True):
self.log += [
AlarmLog(timestamp=datetime.datetime.now(),
from_status=self.status,
to_status=self.status,
message=message)
]
if to_save:
self.save()
def contribute_event(self, e, open=False, close=False):
# Set opening event when necessary
if open:
self.opening_event = e.id
# Set closing event when necessary
if close:
self.closing_event = e.id
# Update timestamp
if e.timestamp < self.timestamp:
self.timestamp = e.timestamp
else:
self.last_update = max(self.last_update, e.timestamp)
self.save()
# Update event's list of alarms
if self.id not in e.alarms:
e.alarms.append(self.id)
e.save()
def clear_alarm(self, message):
ts = datetime.datetime.now()
log = self.log + [
AlarmLog(
timestamp=ts, from_status="A", to_status="C", message=message)
]
a = ArchivedAlarm(id=self.id,
timestamp=self.timestamp,
clear_timestamp=ts,
managed_object=self.managed_object,
alarm_class=self.alarm_class,
severity=self.severity,
vars=self.vars,
log=log,
root=self.root,
opening_event=self.opening_event,
closing_event=self.closing_event,
discriminator=self.discriminator,
reopens=self.reopens)
ct = self.alarm_class.get_control_time(self.reopens)
if ct:
a.control_time = datetime.datetime.now() + datetime.timedelta(
seconds=ct)
a.save()
# @todo: Clear related correlator jobs
self.delete()
# Send notifications
if not a.root and not self.reopens:
a.managed_object.event(
a.managed_object.EV_ALARM_CLEARED, {
"alarm": a,
"subject": a.subject,
"body": a.body,
"symptoms": a.alarm_class.symptoms,
"recommended_actions": a.alarm_class.recommended_actions,
"probable_causes": a.alarm_class.probable_causes
})
elif ct:
# Schedule delayed job
submit_job("fm.correlator",
"control_notify",
key=a.id,
ts=a.control_time)
return a
def get_template_vars(self):
"""
Prepare template variables
"""
vars = self.vars.copy()
vars.update({"alarm": self})
return vars
@property
def subject(self):
ctx = Context(self.get_template_vars())
s = Template(self.alarm_class.subject_template).render(ctx)
if len(s) >= 255:
s = s[:125] + " ... " + s[-125:]
return s
@property
def body(self):
ctx = Context(self.get_template_vars())
s = Template(self.alarm_class.body_template).render(ctx)
return s
def change_owner(self, user):
"""
Change alarm's owner
"""
self.owner = user
self.save()
def subscribe(self, user):
"""
Change alarm's subscribers
"""
if user.id not in self.subscribers:
self.subscribers += [user.id]
self.log_message(
"%s(%s) has been subscribed" %
((" ".join([user.first_name, user.last_name]), user.username)),
to_save=False)
self.save()
def unsubscribe(self, user):
if self.is_subscribed(user):
self.subscribers = [u.id for u in self.subscribers if u != user.id]
self.log_message(
"%s(%s) has been unsubscribed" %
((" ".join([user.first_name, user.last_name]), user.username)),
to_save=False)
self.save()
def is_owner(self, user):
return self.owner == user
def is_subscribed(self, user):
return user.id in self.subscribers
@property
def is_unassigned(self):
return self.owner is None
@property
def duration(self):
dt = datetime.datetime.now() - self.timestamp
return dt.days * 86400 + dt.seconds
@property
def display_duration(self):
duration = datetime.datetime.now() - self.timestamp
secs = duration.seconds % 60
mins = (duration.seconds / 60) % 60
hours = (duration.seconds / 3600) % 24
days = duration.days
r = "%02d:%02d:%02d" % (hours, mins, secs)
if days:
r = "%dd %s" % (days, r)
return r
@property
def effective_style(self):
if self.custom_style:
return self.custom_style
else:
return AlarmSeverity.get_severity(self.severity).style
def set_root(self, root_alarm):
"""
Set root cause
"""
if self.root:
return
if self.id == root_alarm.id:
raise Exception("Cannot set self as root cause")
# Detect loop
root = root_alarm
while root and root.root:
root = root.root
if root == self.id:
return
root = get_alarm(root)
# Set root
self.root = root_alarm.id
self.log_message("Alarm %s has been marked as root cause" %
root_alarm.id)
# self.save() Saved by log_message
root_alarm.log_message("Alarm %s has been marked as child" % self.id)
self._change_root_severity()
# Clear pending notifications
Notification.purge_delayed("alarm:%s" % self.id)
@classmethod
def enable_caching(cls, ttl=600):
cls._fields["alarm_class"].set_cache(ttl)
class ArchivedAlarm(Document):
meta = {
"collection": "noc.alarms.archived",
"strict": False,
"auto_create_index": False,
"indexes": [
"root",
"timestamp",
"managed_object",
("managed_object", "discriminator", "control_time"),
"escalation_tt",
"escalation_ts",
],
}
status = "C"
timestamp = DateTimeField(required=True)
clear_timestamp = DateTimeField(required=True)
managed_object = nosql.ForeignKeyField(ManagedObject)
alarm_class = nosql.PlainReferenceField(AlarmClass)
severity = IntField(required=True)
vars = DictField()
log = ListField(EmbeddedDocumentField(AlarmLog))
#
opening_event = ObjectIdField(required=False)
closing_event = ObjectIdField(required=False)
# Number of reopens
reopens = IntField(required=False)
# Copied discriminator
discriminator = StringField(required=False)
# Manual acknowledgement timestamp
ack_ts = DateTimeField(required=False)
# Manual acknowledgement user name
ack_user = StringField(required=False)
# Control time within alarm will be reopen instead
# instead of creating the new alarm
control_time = DateTimeField(required=False)
# RCA
# Reference to root cause (Active Alarm or Archived Alarm instance)
root = ObjectIdField(required=False)
# Escalated TT ID in form
# <external system name>:<external tt id>
escalation_ts = DateTimeField(required=False)
escalation_tt = StringField(required=False)
escalation_error = StringField(required=False)
escalation_ctx = LongField(required=False)
escalation_close_ts = DateTimeField(required=False)
escalation_close_error = StringField(required=False)
escalation_close_ctx = LongField(required=False)
# Directly affected services summary, grouped by profiles
# (connected to the same managed object)
direct_services = ListField(EmbeddedDocumentField(SummaryItem))
direct_subscribers = ListField(EmbeddedDocumentField(SummaryItem))
# Indirectly affected services summary, groupped by profiles
# (covered by this and all inferred alarms)
total_objects = ListField(EmbeddedDocumentField(ObjectSummaryItem))
total_services = ListField(EmbeddedDocumentField(SummaryItem))
total_subscribers = ListField(EmbeddedDocumentField(SummaryItem))
# Paths
adm_path = ListField(IntField())
segment_path = ListField(ObjectIdField())
container_path = ListField(ObjectIdField())
# Uplinks, for topology_rca only
uplinks = ListField(IntField())
# RCA neighbor cache, for topology_rca only
rca_neighbors = ListField(IntField())
def __str__(self):
return "%s" % self.id
def iter_changed_datastream(self, changed_fields=None):
if config.datastream.enable_alarm:
yield "alarm", self.id
def log_message(self, message):
self.log += [
AlarmLog(
timestamp=datetime.datetime.now(),
from_status=self.status,
to_status=self.status,
message=message,
)
]
self.save()
def get_template_vars(self):
"""
Prepare template variables
"""
vars = self.vars.copy()
vars.update({"alarm": self})
return vars
@property
def subject(self):
ctx = Context(self.get_template_vars())
s = Template(self.alarm_class.subject_template).render(ctx)
if len(s) >= 255:
s = s[:125] + " ... " + s[-125:]
return s
@property
def body(self):
ctx = Context(self.get_template_vars())
s = Template(self.alarm_class.body_template).render(ctx)
return s
@property
def duration(self):
dt = self.clear_timestamp - self.timestamp
return dt.days * 86400 + dt.seconds
@property
def display_duration(self):
duration = self.clear_timestamp - self.timestamp
secs = duration.seconds % 60
mins = (duration.seconds / 60) % 60
hours = (duration.seconds / 3600) % 24
days = duration.days
if days:
return "%dd %02d:%02d:%02d" % (days, hours, mins, secs)
else:
return "%02d:%02d:%02d" % (hours, mins, secs)
@property
def effective_style(self):
return AlarmSeverity.get_severity(self.severity).style
def set_root(self, root_alarm):
pass
def reopen(self, message):
"""
Reopen alarm back
"""
reopens = self.reopens or 0
ts = datetime.datetime.now()
log = self.log + [AlarmLog(timestamp=ts, from_status="C", to_status="A", message=message)]
a = ActiveAlarm(
id=self.id,
timestamp=self.timestamp,
last_update=ts,
managed_object=self.managed_object,
alarm_class=self.alarm_class,
severity=self.severity,
vars=self.vars,
log=log,
root=self.root,
escalation_ts=self.escalation_ts,
escalation_tt=self.escalation_tt,
escalation_error=self.escalation_error,
escalation_ctx=self.escalation_ctx,
opening_event=self.opening_event,
discriminator=self.discriminator,
reopens=reopens + 1,
direct_services=self.direct_services,
direct_subscribers=self.direct_subscribers,
total_objects=self.total_objects,
total_services=self.total_services,
total_subscribers=self.total_subscribers,
adm_path=self.adm_path,
segment_path=self.segment_path,
container_path=self.container_path,
uplinks=self.uplinks,
)
a.save()
# @todo: Clear related correlator jobs
self.delete()
# Send notifications
# Do not set notifications for child and for previously reopened
# alarms
if not a.root and not reopens:
a.managed_object.event(
a.managed_object.EV_ALARM_REOPENED,
{
"alarm": a,
"subject": a.subject,
"body": a.body,
"symptoms": a.alarm_class.symptoms,
"recommended_actions": a.alarm_class.recommended_actions,
"probable_causes": a.alarm_class.probable_causes,
},
)
return a
def iter_consequences(self):
"""
Generator yielding all consequences alarm
"""
for a in ArchivedAlarm.objects.filter(root=self.id):
yield a
for ca in a.iter_consequences():
yield ca
def iter_affected(self):
"""
Generator yielding all affected managed objects
"""
seen = {self.managed_object}
yield self.managed_object
for a in self.iter_consequences():
if a.managed_object not in seen:
seen.add(a.managed_object)
yield a.managed_object
def set_escalation_close_error(self, error):
self.escalation_error = error
self._get_collection().update({"_id": self.id}, {"$set": {"escalation_close_error": error}})
def close_escalation(self):
now = datetime.datetime.now()
self.escalation_close_ts = now
self._get_collection().update({"_id": self.id}, {"$set": {"escalation_close_ts": now}})
def set_escalation_close_ctx(self):
current_context, current_span = get_current_span()
if current_context or self.escalation_close_ctx:
self.escalation_close_ctx = current_context
self._get_collection().update(
{"_id": self.id}, {"$set": {"escalation_close_ctx": current_context}}
)
class ActiveAlarm(nosql.Document):
meta = {
"collection": "noc.alarms.active",
"strict": False,
"auto_create_index": False,
"indexes": [
"timestamp", "root", "-severity",
("alarm_class", "managed_object"),
("discriminator", "managed_object"),
("timestamp", "managed_object"),
"escalation_tt",
"escalation_ts",
"adm_path",
"segment_path",
"container_path",
"uplinks"
]
}
status = "A"
timestamp = nosql.DateTimeField(required=True)
last_update = nosql.DateTimeField(required=True)
managed_object = nosql.ForeignKeyField(ManagedObject)
alarm_class = nosql.PlainReferenceField(AlarmClass)
severity = nosql.IntField(required=True)
vars = nosql.DictField()
# Calculated alarm discriminator
# Has meaning only for alarms with is_unique flag set
# Calculated as sha1("value1\x00....\x00valueN").hexdigest()
discriminator = nosql.StringField(required=False)
log = nosql.ListField(nosql.EmbeddedDocumentField(AlarmLog))
# Responsible person
owner = nosql.ForeignKeyField(User, required=False)
#
opening_event = nosql.ObjectIdField(required=False)
closing_event = nosql.ObjectIdField(required=False)
# List of subscribers
subscribers = nosql.ListField(nosql.ForeignKeyField(User))
#
custom_subject = nosql.StringField(required=False)
custom_style = nosql.ForeignKeyField(Style, required=False)
#
reopens = nosql.IntField(required=False)
# RCA
# Reference to root cause (Active Alarm or Archived Alarm instance)
root = nosql.ObjectIdField(required=False)
# Escalated TT ID in form
# <external system name>:<external tt id>
escalation_ts = nosql.DateTimeField(required=False)
escalation_tt = nosql.StringField(required=False)
escalation_error = nosql.StringField(required=False)
# span context
escalation_ctx = nosql.LongField(required=False)
# Close tt when alarm cleared
close_tt = nosql.BooleanField(default=False)
# Do not clear alarm until *wait_tt* is closed
wait_tt = nosql.StringField()
wait_ts = nosql.DateTimeField()
# Directly affected services summary, grouped by profiles
# (connected to the same managed object)
direct_services = nosql.ListField(nosql.EmbeddedDocumentField(SummaryItem))
direct_subscribers = nosql.ListField(nosql.EmbeddedDocumentField(SummaryItem))
# Indirectly affected services summary, groupped by profiles
# (covered by this and all inferred alarms)
total_objects = nosql.ListField(nosql.EmbeddedDocumentField(ObjectSummaryItem))
total_services = nosql.ListField(nosql.EmbeddedDocumentField(SummaryItem))
total_subscribers = nosql.ListField(nosql.EmbeddedDocumentField(SummaryItem))
# Template and notification group to send close notification
clear_template = nosql.ForeignKeyField(Template, required=False)
clear_notification_group = nosql.ForeignKeyField(NotificationGroup, required=False)
# Paths
adm_path = nosql.ListField(nosql.IntField())
segment_path = nosql.ListField(nosql.ObjectIdField())
container_path = nosql.ListField(nosql.ObjectIdField())
# Uplinks, for topology_rca only
uplinks = nosql.ListField(nosql.IntField())
def __unicode__(self):
return u"%s" % self.id
def iter_changed_datastream(self):
if config.datastream.enable_alarm:
yield "alarm", self.id
def clean(self):
super(ActiveAlarm, self).clean()
if not self.last_update:
self.last_update = self.timestamp
data = self.managed_object.data
self.adm_path = data.adm_path
self.segment_path = data.segment_path
self.container_path = data.container_path
self.uplinks = data.uplinks
def safe_save(self, **kwargs):
"""
Create new alarm or update existing if still exists
:param kwargs:
:return:
"""
if self.id:
# Update existing only if exists
if "save_condition" not in kwargs:
kwargs["save_condition"] = {"id": self.id}
try:
self.save(**kwargs)
except SaveConditionError:
pass # Race condition, closed during update
else:
self.save()
def change_severity(self, user="", delta=None, severity=None, to_save=True):
"""
Change alarm severity
"""
if isinstance(user, User):
user = user.username
if delta:
self.severity = max(0, self.severity + delta)
if delta > 0:
self.log_message(
"%s has increased alarm severity by %s" % (
user, delta))
else:
self.log_message(
"%s has decreased alarm severity by %s" % (
user, delta))
elif severity:
if type(severity) in (int, long, float):
self.severity = int(severity)
self.log_message(
"%s has changed severity to %s" % (user, severity))
else:
self.severity = severity.severity
self.log_message(
"%s has changed severity to %s" % (user, severity.name))
if to_save:
self.safe_save()
def log_message(self, message, to_save=True):
self.log += [AlarmLog(timestamp=datetime.datetime.now(),
from_status=self.status, to_status=self.status,
message=message)]
if to_save:
self.safe_save()
def clear_alarm(self, message, ts=None, force=False):
"""
Clear alarm
:param message: Log clearing message
:param ts: Clearing timestamp
:param force: Clear ever if wait_tt seg
"""
ts = ts or datetime.datetime.now()
if self.wait_tt and not force:
# Wait for escalated tt to close
if not self.wait_ts:
self.wait_ts = ts
self.log_message("Waiting for TT to close")
call_later(
"noc.services.escalator.wait_tt.wait_tt",
scheduler="escalator",
pool=self.managed_object.escalator_shard,
alarm_id=self.id
)
return
if self.alarm_class.clear_handlers:
# Process clear handlers
for h in self.alarm_class.get_clear_handlers():
try:
h(self)
except Exception:
error_report()
log = self.log + [AlarmLog(timestamp=ts, from_status="A",
to_status="C", message=message)]
a = ArchivedAlarm(
id=self.id,
timestamp=self.timestamp,
clear_timestamp=ts,
managed_object=self.managed_object,
alarm_class=self.alarm_class,
severity=self.severity,
vars=self.vars,
log=log,
root=self.root,
escalation_ts=self.escalation_ts,
escalation_tt=self.escalation_tt,
escalation_error=self.escalation_error,
escalation_ctx=self.escalation_ctx,
opening_event=self.opening_event,
closing_event=self.closing_event,
discriminator=self.discriminator,
reopens=self.reopens,
direct_services=self.direct_services,
direct_subscribers=self.direct_subscribers,
total_objects=self.total_objects,
total_services=self.total_services,
total_subscribers=self.total_subscribers,
adm_path=self.adm_path,
segment_path=self.segment_path,
container_path=self.container_path,
uplinks=self.uplinks
)
ct = self.alarm_class.get_control_time(self.reopens)
if ct:
a.control_time = datetime.datetime.now() + datetime.timedelta(seconds=ct)
a.save()
# Send notifications
if not a.root and not self.reopens:
a.managed_object.event(a.managed_object.EV_ALARM_CLEARED, {
"alarm": a,
"subject": a.subject,
"body": a.body,
"symptoms": a.alarm_class.symptoms,
"recommended_actions": a.alarm_class.recommended_actions,
"probable_causes": a.alarm_class.probable_causes
})
elif ct:
pass
# Set checks on all consequences
for d in self._get_collection().find({
"root": self.id
}, {"_id": 1, "alarm_class": 1}):
ac = AlarmClass.get_by_id(d["alarm_class"])
if not ac:
continue
t = ac.recover_time
if not t:
continue
call_later(
"noc.services.correlator.check.check_close_consequence",
scheduler="correlator",
pool=self.managed_object.pool.name,
delay=t,
alarm_id=d["_id"]
)
# Clear alarm
self.delete()
# Close TT
# MUST be after .delete() to prevent race conditions
if a.escalation_tt or self.clear_template:
if self.clear_template:
ctx = {
"alarm": a
}
subject = self.clear_template.render_subject(**ctx)
body = self.clear_template.render_body(**ctx)
else:
subject = "Alarm cleared"
body = "Alarm has been cleared"
call_later(
"noc.services.escalator.escalation.notify_close",
scheduler="escalator",
pool=self.managed_object.escalator_shard,
max_runs=ALARM_CLOSE_RETRIES,
alarm_id=self.id,
tt_id=self.escalation_tt,
subject=subject,
body=body,
notification_group_id=self.clear_notification_group.id if self.clear_notification_group else None,
close_tt=self.close_tt
)
# Gather diagnostics
AlarmDiagnosticConfig.on_clear(a)
# Return archived
return a
def get_template_vars(self):
"""
Prepare template variables
"""
vars = self.vars.copy()
vars.update({"alarm": self})
return vars
@property
def subject(self):
if self.custom_subject:
s = self.custom_subject
else:
ctx = Context(self.get_template_vars())
s = DjangoTemplate(self.alarm_class.subject_template).render(ctx)
if len(s) >= 255:
s = s[:125] + " ... " + s[-125:]
return s
@property
def body(self):
ctx = Context(self.get_template_vars())
s = DjangoTemplate(self.alarm_class.body_template).render(ctx)
return s
def change_owner(self, user):
"""
Change alarm's owner
"""
self.owner = user
self.save()
def subscribe(self, user):
"""
Change alarm's subscribers
"""
if user.id not in self.subscribers:
self.subscribers += [user.id]
self.log_message("%s(%s) has been subscribed" % (
(" ".join([user.first_name, user.last_name]),
user.username)
), to_save=False)
self.save()
def unsubscribe(self, user):
if self.is_subscribed(user):
self.subscribers = [u.id for u in self.subscribers
if u != user.id]
self.log_message("%s(%s) has been unsubscribed" % (
(" ".join([user.first_name, user.last_name]),
user.username)
), to_save=False)
self.save()
def is_owner(self, user):
return self.owner == user
def is_subscribed(self, user):
return user.id in self.subscribers
@property
def is_unassigned(self):
return self.owner is None
@property
def duration(self):
dt = datetime.datetime.now() - self.timestamp
return dt.days * 86400 + dt.seconds
@property
def display_duration(self):
duration = datetime.datetime.now() - self.timestamp
secs = duration.seconds % 60
mins = (duration.seconds / 60) % 60
hours = (duration.seconds / 3600) % 24
days = duration.days
r = "%02d:%02d:%02d" % (hours, mins, secs)
if days:
r = "%dd %s" % (days, r)
return r
@property
def effective_style(self):
if self.custom_style:
return self.custom_style
else:
return AlarmSeverity.get_severity(self.severity).style
def get_root(self):
"""
Get top-level root alarm
"""
root = self
while root.root:
root = get_alarm(root.root)
return root
def update_summary(self):
def update_dict(d1, d2):
for k in d2:
if k in d1:
d1[k] += d2[k]
else:
d1[k] = d2[k]
services = SummaryItem.items_to_dict(self.direct_services)
subscribers = SummaryItem.items_to_dict(self.direct_subscribers)
objects = {
self.managed_object.object_profile.id: 1
}
for a in ActiveAlarm.objects.filter(root=self.id):
a.update_summary()
update_dict(
objects,
SummaryItem.items_to_dict(a.total_objects)
)
update_dict(
services,
SummaryItem.items_to_dict(a.total_services)
)
update_dict(
subscribers,
SummaryItem.items_to_dict(a.total_subscribers)
)
obj_list = ObjectSummaryItem.dict_to_items(objects)
svc_list = SummaryItem.dict_to_items(services)
sub_list = SummaryItem.dict_to_items(subscribers)
if svc_list != self.total_services or sub_list != self.total_subscribers or obj_list != self.total_objects:
ns = ServiceSummary.get_severity({
"service": services,
"subscriber": subscribers,
"objects": objects
})
self.total_objects = obj_list
self.total_services = svc_list
self.total_subscribers = sub_list
if ns != self.severity:
self.change_severity(severity=ns, to_save=False)
self.safe_save()
def set_root(self, root_alarm):
"""
Set root cause
"""
if self.root:
return
if self.id == root_alarm.id:
raise Exception("Cannot set self as root cause")
# Detect loop
root = root_alarm
while root and root.root:
root = root.root
if root == self.id:
return
root = get_alarm(root)
# Set root
self.root = root_alarm.id
self.log_message(
"Alarm %s has been marked as root cause" % root_alarm.id)
# self.save() Saved by log_message
root_alarm.log_message(
"Alarm %s has been marked as child" % self.id)
root_alarm.update_summary()
# Clear pending notifications
# Notification.purge_delayed("alarm:%s" % self.id)
def escalate(self, tt_id, close_tt=False):
self.escalation_tt = tt_id
self.escalation_ts = datetime.datetime.now()
self.close_tt = close_tt
self.log_message("Escalated to %s" % tt_id)
q = {"_id": self.id}
op = {
"$set": {
"escalation_tt": self.escalation_tt,
"escalation_ts": self.escalation_ts,
"close_tt": self.close_tt,
"escalation_error": None
}
}
r = ActiveAlarm._get_collection().update_one(q, op)
if r.acknowledged and not r.modified_count:
# Already closed, update archive
ArchivedAlarm._get_collection().update_one(q, op)
def set_escalation_error(self, error):
self.escalation_error = error
self._get_collection().update_one(
{"_id": self.id},
{"$set": {
"escalation_error": error
}}
)
def set_escalation_context(self):
current_context, current_span = get_current_span()
if current_context or self.escalation_ctx:
self.escalation_ctx = current_context
self._get_collection().update_one(
{"_id": self.id},
{"$set": {
"escalation_ctx": current_context
}}
)
def set_clear_notification(self, notification_group, template):
self.clear_notification_group = notification_group
self.clear_template = template
self.safe_save(save_condition={
"managed_object": {
"$exists": True
},
"id": self.id
})
def iter_consequences(self):
"""
Generator yielding all consequences alarm
"""
for a in ActiveAlarm.objects.filter(root=self.id):
yield a
for ca in a.iter_consequences():
yield ca
def iter_affected(self):
"""
Generator yielding all affected managed objects
"""
seen = set([self.managed_object])
yield self.managed_object
for a in self.iter_consequences():
if a.managed_object not in seen:
seen.add(a.managed_object)
yield a.managed_object
def iter_escalated(self):
"""
Generator yielding all escalated consequences
"""
for a in self.iter_consequences():
if a.escalation_tt:
yield a
class ActiveEvent(document.Document):
"""
Event in the Active state
"""
meta = {
"collection":
"noc.events.active",
"allow_inheritance":
False,
"indexes": [
"timestamp", "discriminator", "alarms",
("timestamp", "event_class", "managed_object")
]
}
status = "A"
# Fields
timestamp = fields.DateTimeField(required=True)
managed_object = nosql.ForeignKeyField(ManagedObject, required=True)
event_class = nosql.PlainReferenceField(EventClass, required=True)
start_timestamp = fields.DateTimeField(required=True)
repeats = fields.IntField(required=True)
raw_vars = nosql.RawDictField()
resolved_vars = nosql.RawDictField()
vars = fields.DictField()
log = fields.ListField(fields.EmbeddedDocumentField(EventLog))
discriminator = fields.StringField(required=False)
alarms = fields.ListField(nosql.ObjectIdField())
def __unicode__(self):
return u"%s" % self.id
def mark_as_new(self, message=None):
"""
Move to new queue
"""
if message is None:
message = "Reclassification requested"
log = self.log + [
EventLog(timestamp=datetime.datetime.now(),
from_status="A",
to_status="N",
message=message)
]
e = NewEvent(id=self.id,
timestamp=self.timestamp,
managed_object=self.managed_object,
raw_vars=self.raw_vars,
log=log)
e.save()
self.delete()
return e
def mark_as_failed(self, version, traceback):
"""
Move event into noc.events.failed
"""
message = "Failed to classify on NOC version %s" % version
log = self.log + [
EventLog(timestamp=datetime.datetime.now(),
from_status="N",
to_status="F",
message=message)
]
e = FailedEvent(id=self.id,
timestamp=self.timestamp,
managed_object=self.managed_object,
raw_vars=self.raw_vars,
version=version,
traceback=traceback,
log=log)
e.save()
self.delete()
return e
def mark_as_archived(self, message):
log = self.log + [
EventLog(timestamp=datetime.datetime.now(),
from_status="A",
to_status="S",
message=message)
]
e = ArchivedEvent(id=self.id,
timestamp=self.timestamp,
managed_object=self.managed_object,
event_class=self.event_class,
start_timestamp=self.start_timestamp,
repeats=self.repeats,
raw_vars=self.raw_vars,
resolved_vars=self.resolved_vars,
vars=self.vars,
log=log,
alarms=self.alarms)
e.save()
self.delete()
return e
def drop(self):
"""
Mark event to be dropped. Only for use from event trigger pyrule.
All further operations on event may lead to unpredictable results.
Event actually deleted by noc-classifier
"""
self.id = None
@property
def to_drop(self):
"""
Check event marked to be dropped
"""
return self.id is None
def log_message(self, message):
self.log += [
EventLog(timestamp=datetime.datetime.now(),
from_status=self.status,
to_status=self.status,
message=message)
]
self.save()
def log_suppression(self, timestamp):
"""
Increate repeat count and update timestamp, if required
"""
self.repeats += 1
if timestamp > self.timestamp:
self.timestamp = timestamp
self.save()
@property
def duration(self):
"""
Logged event duration in seconds
"""
return total_seconds(self.timestamp - self.start_timestamp)
def get_template_vars(self):
"""
Prepare template variables
"""
vars = self.vars.copy()
vars.update({"event": self})
return vars
@property
def subject(self):
ctx = Context(self.get_template_vars())
s = Template(self.event_class.subject_template).render(ctx)
if len(s) >= 255:
s = s[:125] + " ... " + s[-125:]
return s
@property
def body(self):
ctx = Context(self.get_template_vars())
s = Template(self.event_class.body_template).render(ctx)
return s
@property
def managed_object_id(self):
"""
Hack to return managed_object.id without SQL lookup
"""
o = self._data["managed_object"]
if type(o) in (int, long):
return o
return o.id
class ArchivedAlarm(nosql.Document):
meta = {
"collection": "noc.alarms.archived",
"allow_inheritance": False,
"indexes": [
"root",
"control_time",
"timestamp",
"managed_object"
]
}
status = "C"
timestamp = nosql.DateTimeField(required=True)
clear_timestamp = nosql.DateTimeField(required=True)
managed_object = nosql.ForeignKeyField(ManagedObject)
alarm_class = nosql.PlainReferenceField(AlarmClass)
severity = nosql.IntField(required=True)
vars = nosql.DictField()
log = nosql.ListField(nosql.EmbeddedDocumentField(AlarmLog))
#
opening_event = nosql.ObjectIdField(required=False)
closing_event = nosql.ObjectIdField(required=False)
# Number of reopens
reopens = nosql.IntField(required=False)
# Copied discriminator
discriminator = nosql.StringField(required=False)
# Control time within alarm will be reopen instead
# instead of creating the new alarm
control_time = nosql.DateTimeField(required=False)
# RCA
# Reference to root cause (Active Alarm or Archived Alarm instance)
root = nosql.ObjectIdField(required=False)
def __unicode__(self):
return u"%s" % self.id
def log_message(self, message):
self.log += [AlarmLog(timestamp=datetime.datetime.now(),
from_status=self.status, to_status=self.status,
message=message)]
self.save()
def get_template_vars(self):
"""
Prepare template variables
"""
vars = self.vars.copy()
vars.update({"event": self})
return vars
@property
def subject(self):
ctx = Context(self.get_template_vars())
s = Template(self.alarm_class.subject_template).render(ctx)
if len(s) >= 255:
s = s[:125] + " ... " + s[-125:]
return s
@property
def body(self):
ctx = Context(self.get_template_vars())
s = Template(self.alarm_class.body_template).render(ctx)
return s
@property
def duration(self):
dt = self.clear_timestamp - self.timestamp
return dt.days * 86400 + dt.seconds
@property
def display_duration(self):
duration = self.clear_timestamp - self.timestamp
secs = duration.seconds % 60
mins = (duration.seconds / 60) % 60
hours = (duration.seconds / 3600) % 24
days = duration.days
if days:
return "%dd %02d:%02d:%02d" % (days, hours, mins, secs)
else:
return "%02d:%02d:%02d" % (hours, mins, secs)
@property
def effective_style(self):
return AlarmSeverity.get_severity(self.severity).style
def set_root(self, root_alarm):
pass
def reopen(self, message):
"""
Reopen alarm back
"""
reopens = self.reopens or 0
ts = datetime.datetime.now()
log = self.log + [AlarmLog(timestamp=ts, from_status="C",
to_status="A", message=message)]
a = ActiveAlarm(
id=self.id,
timestamp=self.timestamp,
last_update=ts,
managed_object=self.managed_object,
alarm_class=self.alarm_class,
severity=self.severity,
vars=self.vars,
log=log,
root=self.root,
opening_event=self.opening_event,
discriminator=self.discriminator,
reopens=reopens + 1
)
a.save()
# @todo: Clear related correlator jobs
self.delete()
# Remove pending control_notify job
remove_job("fm.correlator", "control_notify", key=a.id)
# Send notifications
# Do not set notifications for child and for previously reopened
# alarms
if not a.root and not reopens:
a.managed_object.event(a.managed_object.EV_ALARM_REOPENED, {
"alarm": a,
"subject": a.subject,
"body": a.body,
"symptoms": a.alarm_class.symptoms,
"recommended_actions": a.alarm_class.recommended_actions,
"probable_causes": a.alarm_class.probable_causes
})
return a