class Node(nosql.Document):
meta = {"collection": "noc.wf.nodes", "allow_inheritance": False}
workflow = nosql.PlainReferenceField(Workflow)
lane = nosql.PlainReferenceField(Lane)
name = nosql.StringField()
label = nosql.StringField()
description = nosql.StringField()
handler = nosql.StringField()
# param -> value
params = nosql.RawDictField()
# Connections
next_node = nosql.PlainReferenceField("self")
next_true_node = nosql.PlainReferenceField("self")
next_false_node = nosql.PlainReferenceField("self")
# Graph position
x = nosql.IntField()
y = nosql.IntField()
def __unicode__(self):
return "%s %s" % (self.workflow, self.name)
@property
def handler_class(self):
m = __import__("noc.wf.handlers", {}, {}, str(self.handler))
return getattr(m, "%sHandler" % self.handler)
class AlarmRootCauseCondition(nosql.EmbeddedDocument):
meta = {
"strict": False,
"auto_create_index": False
}
name = nosql.StringField(required=True)
root = nosql.PlainReferenceField("fm.AlarmClass")
window = nosql.IntField(required=True)
condition = nosql.StringField(default="True")
match_condition = nosql.DictField(required=True)
def __unicode__(self):
return self.name
def __eq__(self, other):
return (
self.name == other.name and
(
(self.root is None and other.root is None) or
(
self.root and other.root and
self.root.id == other.root.id
)
) and
self.window == other.window and
self.condition == other.condition and
self.match_condition == other.match_condition
)
class Overlay(nosql.Document):
meta = {"collection": "noc.gis.overlays", "allow_inheritance": False}
name = nosql.StringField(required=True)
gate_id = nosql.StringField(unique=True)
is_active = nosql.BooleanField(required=True, default=True)
permission_name = nosql.StringField(required=True)
overlay = nosql.StringField(required=True)
config = nosql.DictField()
_overlay_cache = {} # name -> Overlay
def __unicode__(self):
return self.name
def get_overlay(self):
if self.overlay not in self._overlay_cache:
from noc.gis.overlays.base import OverlayHandler
m = __import__("noc.gis.overlays.%s" % self.overlay, {}, {}, "*")
for n in dir(m):
o = getattr(m, n)
if (inspect.isclass(o) and o != OverlayHandler
and issubclass(o, OverlayHandler)):
self._overlay_cache[self.overlay] = o
break
h = self._overlay_cache[self.overlay]
return h(**self.config)
class Workflow(nosql.Document):
meta = {"collection": "noc.wf.workflows", "allow_inheritance": False}
# Unique identifier
name = nosql.StringField()
# Long name
display_name = nosql.StringField()
solution = nosql.PlainReferenceField(Solution)
version = nosql.IntField()
is_active = nosql.BooleanField()
description = nosql.StringField()
#
start_node = nosql.StringField()
# Permissions
# stat_permission = nosql.StringField()
# trace_permission = nosql.StringField()
# kill_permission = nosql.StringField()
trace = nosql.BooleanField(default=False)
def __unicode__(self):
return "%s.%s v%s" % (self.solution.name, self.name, self.version)
def get_node(self, name):
return Node.objects.filter(workflow=self.id, name=name).first()
def get_start_node(self):
return Node.objects.filter(workflow=self.id,
id=self.start_node).first()
def run(self, _trace=None, **kwargs):
"""
Run process
:param kwargs:
:return: Process instance
"""
# Find start node
start_node = self.get_start_node()
if not start_node:
raise InvalidStartNode(self.start_node)
#
trace = self.trace if _trace is None else _trace
# Prepare context
ctx = {}
for v in Variable.objects.filter(workflow=self.id):
if v.name in kwargs:
ctx[v.name] = v.clean(kwargs[v.name])
elif v.default:
ctx[v.name] = v.clean(v.default)
else:
ctx[v.name] = None
p = Process(workflow=self,
context=ctx,
start_time=datetime.datetime.now(),
node=start_node,
trace=trace)
p.save()
# Schedule job
p.schedule()
return p
class Variable(nosql.Document):
meta = {"collection": "noc.wf.variables", "allow_inheritance": False}
workflow = nosql.PlainReferenceField(Workflow)
name = nosql.StringField()
type = nosql.StringField(
choices=[("str", "String"), ("int",
"Integer"), ("bool",
"Boolean"), ("float",
"Float")])
default = nosql.StringField()
# Required to start the process
required = nosql.BooleanField()
description = nosql.StringField()
def __unicode__(self):
return "%s %s (%s)" % (self.workflow, self.name, self.type)
def clean(self, value):
return getattr(self, "clean_%s" % self.type)(value)
def clean_str(self, value):
return value
def clean_int(self, value):
return int(value)
def clean_bool(self, value):
return value.lower() in ["on", "true", "yes"]
def clean_float(self, value):
return float(value)
class Map(nosql.Document):
meta = {
"collection": "noc.gis.maps",
"strict": False,
"auto_create_index": False
}
name = nosql.StringField(unique=True)
is_builtin = nosql.BooleanField(default=True)
is_active = nosql.BooleanField(default=True)
# srs = nosql.ForeignKeyField(SRS)
layers = nosql.ListField(nosql.StringField())
def __unicode__(self):
return self.name
@property
def active_layers(self):
"""
Get list of active layers
:return:
"""
r = []
for ln in self.layers:
l = Layer.objects.filter(name=ln).first()
if l and l.is_active:
r += [l]
return r
class UserState(nosql.Document):
meta = {"collection": "noc.userstate", "allow_inheritance": False}
user_id = nosql.IntField()
key = nosql.StringField()
value = nosql.StringField()
def __unicode__(self):
return "%s: %s" % (self.user_id, self.key)
class FontSet(nosql.Document):
meta = {"collection": "noc.gis.fontsets", "allow_inheritance": False}
name = nosql.StringField(unique=True)
is_builtin = nosql.BooleanField(default=True)
description = nosql.StringField(required=False)
fonts = nosql.ListField(nosql.StringField())
def __unicode__(self):
return self.name
class Layer(nosql.Document):
meta = {"collection": "noc.gis.layers", "allow_inheritance": False}
name = nosql.StringField(unique=True)
is_builtin = nosql.BooleanField(default=True)
is_active = nosql.BooleanField(default=True)
#srs = nosql.ForeignKeyField(SRS)
styles = nosql.ListField(nosql.StringField())
datasource = nosql.DictField()
def __unicode__(self):
return self.name
class DataSource(nosql.EmbeddedDocument):
meta = {"allow_inheritance": False}
name = nosql.StringField()
datasource = nosql.StringField()
search = nosql.DictField()
def __unicode__(self):
return self.name
def __eq__(self, other):
return (self.name == other.name and self.datasource == other.datasource
and self.search == other.search)
class DataSource(nosql.EmbeddedDocument):
meta = {"strict": False, "auto_create_index": False}
name = nosql.StringField()
datasource = nosql.StringField()
search = nosql.DictField()
def __unicode__(self):
return self.name
def __eq__(self, other):
return (self.name == other.name and self.datasource == other.datasource
and self.search == other.search)
class AlarmLog(nosql.EmbeddedDocument):
meta = {"allow_inheritance": False}
timestamp = nosql.DateTimeField()
from_status = nosql.StringField(max_length=1,
regex=r"^[AC]$",
required=True)
to_status = nosql.StringField(max_length=1, regex=r"^[AC]$", required=True)
message = nosql.StringField()
def __unicode__(self):
return u"%s [%s -> %s]: %s" % (self.timestamp, self.from_status,
self.to_status, self.message)
class AlarmClassVar(nosql.EmbeddedDocument):
meta = {"allow_inheritance": False}
name = nosql.StringField(required=True)
description = nosql.StringField(required=False)
default = nosql.StringField(required=False)
def __unicode__(self):
return self.name
def __eq__(self, other):
return (self.name == other.name
and self.description == other.description
and self.default == other.default)
class AlarmClassVar(nosql.EmbeddedDocument):
meta = {"strict": False, "auto_create_index": False}
name = nosql.StringField(required=True)
description = nosql.StringField(required=False)
default = nosql.StringField(required=False)
def __unicode__(self):
return self.name
def __eq__(self, other):
return (self.name == other.name
and self.description == other.description
and self.default == other.default)
class Area(nosql.Document):
meta = {"allow_inheritance": False, "collection": "noc.gis.areas"}
name = nosql.StringField()
is_active = nosql.BooleanField(default=True)
min_zoom = nosql.IntField(default=0)
max_zoom = nosql.IntField(default=18)
# (EPSG:4326) coordinates
SW = nosql.GeoPointField()
NE = nosql.GeoPointField()
description = nosql.StringField(required=False)
def __unicode__(self):
return self.name
class MIBData(nosql.Document):
meta = {
"collection": "noc.mibdata",
"allow_inheritance": False,
"indexes": ["oid", "name", "mib", "aliases"]
}
mib = nosql.PlainReferenceField(MIB)
oid = nosql.StringField(required=True, unique=True)
name = nosql.StringField(required=True)
description = nosql.StringField(required=False)
syntax = nosql.DictField(required=False)
aliases = nosql.ListField(nosql.StringField(), default=[])
def __unicode__(self):
return self.name
class AlarmLog(nosql.EmbeddedDocument):
meta = {"strict": False, "auto_create_index": False}
timestamp = nosql.DateTimeField()
from_status = nosql.StringField(max_length=1,
regex=r"^[AC]$",
required=True)
to_status = nosql.StringField(max_length=1, regex=r"^[AC]$", required=True)
message = nosql.StringField()
def __str__(self):
return "%s [%s -> %s]: %s" % (
self.timestamp,
self.from_status,
self.to_status,
self.message,
)
class Style(nosql.Document):
meta = {"collection": "noc.gis.styles", "allow_inheritance": False}
name = nosql.StringField(unique=True)
is_builtin = nosql.BooleanField(default=True)
rules = nosql.ListField(nosql.EmbeddedDocumentField(Rule))
def __unicode__(self):
return self.name
class Lane(nosql.Document):
meta = {"collection": "noc.wf.lanes", "allow_inheritance": False}
workflow = nosql.PlainReferenceField(Workflow)
name = nosql.StringField()
is_active = nosql.BooleanField()
def __unicode__(self):
return "%s %s" % (self.workflow, self.name)
class Solution(nosql.Document):
meta = {"collection": "noc.wf.solutions", "allow_inheritance": False}
name = nosql.StringField()
version = nosql.IntField(default=1)
is_active = nosql.BooleanField(default=False)
description = nosql.StringField()
def __unicode__(self):
return "%s v%s" % (self.name, self.version)
@classmethod
def get_active(cls, name):
"""
Get last version of active solution
:param cls:
:param name: Solution name
:return: Solution or None
"""
return cls.objects.filter(name=name, is_active=True)\
.order_by("-version").first()
class Rule(nosql.EmbeddedDocument):
meta = {"allow_inheritance": False}
minscale_zoom = nosql.IntField(required=False)
maxscale_zoom = nosql.IntField(required=False)
rule_filter = nosql.StringField(required=False)
symbolizers = nosql.ListField(nosql.DictField())
def __unicode__(self):
return unicode(id(self))
def __eq__(self, other):
return (self.minscale_zoom == other.minscale_zoom
and self.maxscale_zoom == other.maxscale_zoom
and self.rule_filter == other.rule_filter
and self.symbolizers == other.symbolizers)
class AlarmClassJob(nosql.EmbeddedDocument):
meta = {"allow_inheritance": False}
# Job name (fm/correlator/jobs/<name>.py)
job = nosql.StringField(required=True)
# Start job after *delay* seconds after alarm risen
# delay = nosql.IntField(required=False, default=0)
# Restart job every *interval* seconds
interval = nosql.IntField(required=False, default=0)
# Job parameters: name -> expression
vars = nosql.DictField(required=True)
def __unicode__(self):
return self.job
def __eq__(self, other):
return (self.job == other.job and self.interval == other.interval
and self.vars == other.vars)
class UserSession(nosql.Document):
meta = {"collection": "noc.user_sessions", "allow_inheritance": False}
session_key = nosql.StringField(primary_key=True)
user_id = nosql.IntField()
@classmethod
def register(cls, session_key, user):
UserSession(session_key=session_key,
user_id=user.id).save(force_insert=True)
@classmethod
def unregister(cls, session_key):
UserSession.objects.filter(session_key=session_key).delete()
@classmethod
def active_sessions(cls, user=None, group=None):
"""
Calculate current active sessions for user and group
"""
ids = []
if user:
ids += [user.id]
if group:
ids += group.user_set.values_list("id", flat=True)
n = 0
now = datetime.datetime.now()
for us in UserSession.objects.filter(user_id__in=ids):
s = MongoSession.objects.filter(session_key=us.session_key).first()
if s:
# Session exists
if s.expire_date < now:
# Expired session
s.delete()
else:
n += 1 # Count as active
else:
# Hanging session, schedule to kill
us.delete()
return n
class AlarmClassCategory(nosql.Document):
meta = {
"collection": "noc.alartmclasscategories", # @todo: Fix bug
"allow_inheritance": False
}
name = nosql.StringField()
parent = nosql.ObjectIdField(required=False)
def __unicode__(self):
return self.name
def save(self, *args, **kwargs):
if " | " in self.name:
p_name = " | ".join(self.name.split(" | ")[:-1])
p = AlarmClassCategory.objects.filter(name=p_name).first()
if not p:
p = AlarmClassCategory(name=p_name)
p.save()
self.parent = p.id
else:
self.parent = None
super(AlarmClassCategory, self).save(*args, **kwargs)
class ActiveAlarm(nosql.Document):
meta = {
"collection": "noc.alarms.active",
"strict": False,
"auto_create_index": False,
"indexes": [
"timestamp", "root", "-severity",
("alarm_class", "managed_object"),
("discriminator", "managed_object"),
("timestamp", "managed_object"),
"escalation_tt",
"escalation_ts",
"adm_path",
"segment_path",
"container_path",
"uplinks"
]
}
status = "A"
timestamp = nosql.DateTimeField(required=True)
last_update = nosql.DateTimeField(required=True)
managed_object = nosql.ForeignKeyField(ManagedObject)
alarm_class = nosql.PlainReferenceField(AlarmClass)
severity = nosql.IntField(required=True)
vars = nosql.DictField()
# Calculated alarm discriminator
# Has meaning only for alarms with is_unique flag set
# Calculated as sha1("value1\x00....\x00valueN").hexdigest()
discriminator = nosql.StringField(required=False)
log = nosql.ListField(nosql.EmbeddedDocumentField(AlarmLog))
# Responsible person
owner = nosql.ForeignKeyField(User, required=False)
#
opening_event = nosql.ObjectIdField(required=False)
closing_event = nosql.ObjectIdField(required=False)
# List of subscribers
subscribers = nosql.ListField(nosql.ForeignKeyField(User))
#
custom_subject = nosql.StringField(required=False)
custom_style = nosql.ForeignKeyField(Style, required=False)
#
reopens = nosql.IntField(required=False)
# RCA
# Reference to root cause (Active Alarm or Archived Alarm instance)
root = nosql.ObjectIdField(required=False)
# Escalated TT ID in form
# <external system name>:<external tt id>
escalation_ts = nosql.DateTimeField(required=False)
escalation_tt = nosql.StringField(required=False)
escalation_error = nosql.StringField(required=False)
# span context
escalation_ctx = nosql.LongField(required=False)
# Close tt when alarm cleared
close_tt = nosql.BooleanField(default=False)
# Do not clear alarm until *wait_tt* is closed
wait_tt = nosql.StringField()
wait_ts = nosql.DateTimeField()
# Directly affected services summary, grouped by profiles
# (connected to the same managed object)
direct_services = nosql.ListField(nosql.EmbeddedDocumentField(SummaryItem))
direct_subscribers = nosql.ListField(nosql.EmbeddedDocumentField(SummaryItem))
# Indirectly affected services summary, groupped by profiles
# (covered by this and all inferred alarms)
total_objects = nosql.ListField(nosql.EmbeddedDocumentField(ObjectSummaryItem))
total_services = nosql.ListField(nosql.EmbeddedDocumentField(SummaryItem))
total_subscribers = nosql.ListField(nosql.EmbeddedDocumentField(SummaryItem))
# Template and notification group to send close notification
clear_template = nosql.ForeignKeyField(Template, required=False)
clear_notification_group = nosql.ForeignKeyField(NotificationGroup, required=False)
# Paths
adm_path = nosql.ListField(nosql.IntField())
segment_path = nosql.ListField(nosql.ObjectIdField())
container_path = nosql.ListField(nosql.ObjectIdField())
# Uplinks, for topology_rca only
uplinks = nosql.ListField(nosql.IntField())
def __unicode__(self):
return u"%s" % self.id
def iter_changed_datastream(self):
if config.datastream.enable_alarm:
yield "alarm", self.id
def clean(self):
super(ActiveAlarm, self).clean()
if not self.last_update:
self.last_update = self.timestamp
data = self.managed_object.data
self.adm_path = data.adm_path
self.segment_path = data.segment_path
self.container_path = data.container_path
self.uplinks = data.uplinks
def safe_save(self, **kwargs):
"""
Create new alarm or update existing if still exists
:param kwargs:
:return:
"""
if self.id:
# Update existing only if exists
if "save_condition" not in kwargs:
kwargs["save_condition"] = {"id": self.id}
try:
self.save(**kwargs)
except SaveConditionError:
pass # Race condition, closed during update
else:
self.save()
def change_severity(self, user="", delta=None, severity=None, to_save=True):
"""
Change alarm severity
"""
if isinstance(user, User):
user = user.username
if delta:
self.severity = max(0, self.severity + delta)
if delta > 0:
self.log_message(
"%s has increased alarm severity by %s" % (
user, delta))
else:
self.log_message(
"%s has decreased alarm severity by %s" % (
user, delta))
elif severity:
if type(severity) in (int, long, float):
self.severity = int(severity)
self.log_message(
"%s has changed severity to %s" % (user, severity))
else:
self.severity = severity.severity
self.log_message(
"%s has changed severity to %s" % (user, severity.name))
if to_save:
self.safe_save()
def log_message(self, message, to_save=True):
self.log += [AlarmLog(timestamp=datetime.datetime.now(),
from_status=self.status, to_status=self.status,
message=message)]
if to_save:
self.safe_save()
def clear_alarm(self, message, ts=None, force=False):
"""
Clear alarm
:param message: Log clearing message
:param ts: Clearing timestamp
:param force: Clear ever if wait_tt seg
"""
ts = ts or datetime.datetime.now()
if self.wait_tt and not force:
# Wait for escalated tt to close
if not self.wait_ts:
self.wait_ts = ts
self.log_message("Waiting for TT to close")
call_later(
"noc.services.escalator.wait_tt.wait_tt",
scheduler="escalator",
pool=self.managed_object.escalator_shard,
alarm_id=self.id
)
return
if self.alarm_class.clear_handlers:
# Process clear handlers
for h in self.alarm_class.get_clear_handlers():
try:
h(self)
except Exception:
error_report()
log = self.log + [AlarmLog(timestamp=ts, from_status="A",
to_status="C", message=message)]
a = ArchivedAlarm(
id=self.id,
timestamp=self.timestamp,
clear_timestamp=ts,
managed_object=self.managed_object,
alarm_class=self.alarm_class,
severity=self.severity,
vars=self.vars,
log=log,
root=self.root,
escalation_ts=self.escalation_ts,
escalation_tt=self.escalation_tt,
escalation_error=self.escalation_error,
escalation_ctx=self.escalation_ctx,
opening_event=self.opening_event,
closing_event=self.closing_event,
discriminator=self.discriminator,
reopens=self.reopens,
direct_services=self.direct_services,
direct_subscribers=self.direct_subscribers,
total_objects=self.total_objects,
total_services=self.total_services,
total_subscribers=self.total_subscribers,
adm_path=self.adm_path,
segment_path=self.segment_path,
container_path=self.container_path,
uplinks=self.uplinks
)
ct = self.alarm_class.get_control_time(self.reopens)
if ct:
a.control_time = datetime.datetime.now() + datetime.timedelta(seconds=ct)
a.save()
# Send notifications
if not a.root and not self.reopens:
a.managed_object.event(a.managed_object.EV_ALARM_CLEARED, {
"alarm": a,
"subject": a.subject,
"body": a.body,
"symptoms": a.alarm_class.symptoms,
"recommended_actions": a.alarm_class.recommended_actions,
"probable_causes": a.alarm_class.probable_causes
})
elif ct:
pass
# Set checks on all consequences
for d in self._get_collection().find({
"root": self.id
}, {"_id": 1, "alarm_class": 1}):
ac = AlarmClass.get_by_id(d["alarm_class"])
if not ac:
continue
t = ac.recover_time
if not t:
continue
call_later(
"noc.services.correlator.check.check_close_consequence",
scheduler="correlator",
pool=self.managed_object.pool.name,
delay=t,
alarm_id=d["_id"]
)
# Clear alarm
self.delete()
# Close TT
# MUST be after .delete() to prevent race conditions
if a.escalation_tt or self.clear_template:
if self.clear_template:
ctx = {
"alarm": a
}
subject = self.clear_template.render_subject(**ctx)
body = self.clear_template.render_body(**ctx)
else:
subject = "Alarm cleared"
body = "Alarm has been cleared"
call_later(
"noc.services.escalator.escalation.notify_close",
scheduler="escalator",
pool=self.managed_object.escalator_shard,
max_runs=ALARM_CLOSE_RETRIES,
alarm_id=self.id,
tt_id=self.escalation_tt,
subject=subject,
body=body,
notification_group_id=self.clear_notification_group.id if self.clear_notification_group else None,
close_tt=self.close_tt
)
# Gather diagnostics
AlarmDiagnosticConfig.on_clear(a)
# Return archived
return a
def get_template_vars(self):
"""
Prepare template variables
"""
vars = self.vars.copy()
vars.update({"alarm": self})
return vars
@property
def subject(self):
if self.custom_subject:
s = self.custom_subject
else:
ctx = Context(self.get_template_vars())
s = DjangoTemplate(self.alarm_class.subject_template).render(ctx)
if len(s) >= 255:
s = s[:125] + " ... " + s[-125:]
return s
@property
def body(self):
ctx = Context(self.get_template_vars())
s = DjangoTemplate(self.alarm_class.body_template).render(ctx)
return s
def change_owner(self, user):
"""
Change alarm's owner
"""
self.owner = user
self.save()
def subscribe(self, user):
"""
Change alarm's subscribers
"""
if user.id not in self.subscribers:
self.subscribers += [user.id]
self.log_message("%s(%s) has been subscribed" % (
(" ".join([user.first_name, user.last_name]),
user.username)
), to_save=False)
self.save()
def unsubscribe(self, user):
if self.is_subscribed(user):
self.subscribers = [u.id for u in self.subscribers
if u != user.id]
self.log_message("%s(%s) has been unsubscribed" % (
(" ".join([user.first_name, user.last_name]),
user.username)
), to_save=False)
self.save()
def is_owner(self, user):
return self.owner == user
def is_subscribed(self, user):
return user.id in self.subscribers
@property
def is_unassigned(self):
return self.owner is None
@property
def duration(self):
dt = datetime.datetime.now() - self.timestamp
return dt.days * 86400 + dt.seconds
@property
def display_duration(self):
duration = datetime.datetime.now() - self.timestamp
secs = duration.seconds % 60
mins = (duration.seconds / 60) % 60
hours = (duration.seconds / 3600) % 24
days = duration.days
r = "%02d:%02d:%02d" % (hours, mins, secs)
if days:
r = "%dd %s" % (days, r)
return r
@property
def effective_style(self):
if self.custom_style:
return self.custom_style
else:
return AlarmSeverity.get_severity(self.severity).style
def get_root(self):
"""
Get top-level root alarm
"""
root = self
while root.root:
root = get_alarm(root.root)
return root
def update_summary(self):
def update_dict(d1, d2):
for k in d2:
if k in d1:
d1[k] += d2[k]
else:
d1[k] = d2[k]
services = SummaryItem.items_to_dict(self.direct_services)
subscribers = SummaryItem.items_to_dict(self.direct_subscribers)
objects = {
self.managed_object.object_profile.id: 1
}
for a in ActiveAlarm.objects.filter(root=self.id):
a.update_summary()
update_dict(
objects,
SummaryItem.items_to_dict(a.total_objects)
)
update_dict(
services,
SummaryItem.items_to_dict(a.total_services)
)
update_dict(
subscribers,
SummaryItem.items_to_dict(a.total_subscribers)
)
obj_list = ObjectSummaryItem.dict_to_items(objects)
svc_list = SummaryItem.dict_to_items(services)
sub_list = SummaryItem.dict_to_items(subscribers)
if svc_list != self.total_services or sub_list != self.total_subscribers or obj_list != self.total_objects:
ns = ServiceSummary.get_severity({
"service": services,
"subscriber": subscribers,
"objects": objects
})
self.total_objects = obj_list
self.total_services = svc_list
self.total_subscribers = sub_list
if ns != self.severity:
self.change_severity(severity=ns, to_save=False)
self.safe_save()
def set_root(self, root_alarm):
"""
Set root cause
"""
if self.root:
return
if self.id == root_alarm.id:
raise Exception("Cannot set self as root cause")
# Detect loop
root = root_alarm
while root and root.root:
root = root.root
if root == self.id:
return
root = get_alarm(root)
# Set root
self.root = root_alarm.id
self.log_message(
"Alarm %s has been marked as root cause" % root_alarm.id)
# self.save() Saved by log_message
root_alarm.log_message(
"Alarm %s has been marked as child" % self.id)
root_alarm.update_summary()
# Clear pending notifications
# Notification.purge_delayed("alarm:%s" % self.id)
def escalate(self, tt_id, close_tt=False):
self.escalation_tt = tt_id
self.escalation_ts = datetime.datetime.now()
self.close_tt = close_tt
self.log_message("Escalated to %s" % tt_id)
q = {"_id": self.id}
op = {
"$set": {
"escalation_tt": self.escalation_tt,
"escalation_ts": self.escalation_ts,
"close_tt": self.close_tt,
"escalation_error": None
}
}
r = ActiveAlarm._get_collection().update_one(q, op)
if r.acknowledged and not r.modified_count:
# Already closed, update archive
ArchivedAlarm._get_collection().update_one(q, op)
def set_escalation_error(self, error):
self.escalation_error = error
self._get_collection().update_one(
{"_id": self.id},
{"$set": {
"escalation_error": error
}}
)
def set_escalation_context(self):
current_context, current_span = get_current_span()
if current_context or self.escalation_ctx:
self.escalation_ctx = current_context
self._get_collection().update_one(
{"_id": self.id},
{"$set": {
"escalation_ctx": current_context
}}
)
def set_clear_notification(self, notification_group, template):
self.clear_notification_group = notification_group
self.clear_template = template
self.safe_save(save_condition={
"managed_object": {
"$exists": True
},
"id": self.id
})
def iter_consequences(self):
"""
Generator yielding all consequences alarm
"""
for a in ActiveAlarm.objects.filter(root=self.id):
yield a
for ca in a.iter_consequences():
yield ca
def iter_affected(self):
"""
Generator yielding all affected managed objects
"""
seen = set([self.managed_object])
yield self.managed_object
for a in self.iter_consequences():
if a.managed_object not in seen:
seen.add(a.managed_object)
yield a.managed_object
def iter_escalated(self):
"""
Generator yielding all escalated consequences
"""
for a in self.iter_consequences():
if a.escalation_tt:
yield a
class ActiveAlarm(nosql.Document):
meta = {
"collection":
"noc.alarms.active",
"allow_inheritance":
False,
"indexes": [
"timestamp", "discriminator", "root", "-severity", "alarm_class",
("timestamp", "managed_object")
]
}
status = "A"
timestamp = nosql.DateTimeField(required=True)
last_update = nosql.DateTimeField(required=True)
managed_object = nosql.ForeignKeyField(ManagedObject)
alarm_class = nosql.PlainReferenceField(AlarmClass)
severity = nosql.IntField(required=True)
vars = nosql.DictField()
# Calculated alarm discriminator
# Has meaning only for alarms with is_unique flag set
# Calculated as sha1("value1\x00....\x00valueN").hexdigest()
discriminator = nosql.StringField(required=False)
log = nosql.ListField(nosql.EmbeddedDocumentField(AlarmLog))
# Responsible person
owner = nosql.ForeignKeyField(User, required=False)
#
opening_event = nosql.ObjectIdField(required=False)
closing_event = nosql.ObjectIdField(required=False)
# List of subscribers
subscribers = nosql.ListField(nosql.ForeignKeyField(User))
#
custom_subject = nosql.StringField(required=False)
custom_style = nosql.ForeignKeyField(Style, required=False)
#
reopens = nosql.IntField(required=False)
# RCA
# Reference to root cause (Active Alarm or Archived Alarm instance)
root = nosql.ObjectIdField(required=False)
def __unicode__(self):
return u"%s" % self.id
def save(self, *args, **kwargs):
if not self.last_update:
self.last_update = self.timestamp
return super(ActiveAlarm, self).save(*args, **kwargs)
def _change_root_severity(self):
"""
Change root severity, when necessary
"""
if not self.root:
return
root = get_alarm(self.root)
if root and root.severity < self.severity:
root.change_severity(self.severity)
root.log_message("Severity has been increased by child alarm %s" %
self.id)
def change_severity(self, user="", delta=None, severity=None):
"""
Change alarm severity
"""
if isinstance(user, User):
user = user.username
if delta:
self.severity = max(0, self.severity + delta)
if delta > 0:
self.log_message("%s has increased alarm severity by %s" %
(user, delta))
else:
self.log_message("%s has decreased alarm severity by %s" %
(user, delta))
elif severity:
self.severity = severity.severity
self.log_message("%s has changed severity to %s" %
(user, severity.name))
self._change_root_severity()
self.save()
def log_message(self, message, to_save=True):
self.log += [
AlarmLog(timestamp=datetime.datetime.now(),
from_status=self.status,
to_status=self.status,
message=message)
]
if to_save:
self.save()
def contribute_event(self, e, open=False, close=False):
# Set opening event when necessary
if open:
self.opening_event = e.id
# Set closing event when necessary
if close:
self.closing_event = e.id
# Update timestamp
if e.timestamp < self.timestamp:
self.timestamp = e.timestamp
else:
self.last_update = max(self.last_update, e.timestamp)
self.save()
# Update event's list of alarms
if self.id not in e.alarms:
e.alarms.append(self.id)
e.save()
def clear_alarm(self, message):
ts = datetime.datetime.now()
log = self.log + [
AlarmLog(
timestamp=ts, from_status="A", to_status="C", message=message)
]
a = ArchivedAlarm(id=self.id,
timestamp=self.timestamp,
clear_timestamp=ts,
managed_object=self.managed_object,
alarm_class=self.alarm_class,
severity=self.severity,
vars=self.vars,
log=log,
root=self.root,
opening_event=self.opening_event,
closing_event=self.closing_event,
discriminator=self.discriminator,
reopens=self.reopens)
ct = self.alarm_class.get_control_time(self.reopens)
if ct:
a.control_time = datetime.datetime.now() + datetime.timedelta(
seconds=ct)
a.save()
# @todo: Clear related correlator jobs
self.delete()
# Send notifications
if not a.root and not self.reopens:
a.managed_object.event(
a.managed_object.EV_ALARM_CLEARED, {
"alarm": a,
"subject": a.subject,
"body": a.body,
"symptoms": a.alarm_class.symptoms,
"recommended_actions": a.alarm_class.recommended_actions,
"probable_causes": a.alarm_class.probable_causes
})
elif ct:
# Schedule delayed job
submit_job("fm.correlator",
"control_notify",
key=a.id,
ts=a.control_time)
return a
def get_template_vars(self):
"""
Prepare template variables
"""
vars = self.vars.copy()
vars.update({"alarm": self})
return vars
@property
def subject(self):
ctx = Context(self.get_template_vars())
s = Template(self.alarm_class.subject_template).render(ctx)
if len(s) >= 255:
s = s[:125] + " ... " + s[-125:]
return s
@property
def body(self):
ctx = Context(self.get_template_vars())
s = Template(self.alarm_class.body_template).render(ctx)
return s
def change_owner(self, user):
"""
Change alarm's owner
"""
self.owner = user
self.save()
def subscribe(self, user):
"""
Change alarm's subscribers
"""
if user.id not in self.subscribers:
self.subscribers += [user.id]
self.log_message(
"%s(%s) has been subscribed" %
((" ".join([user.first_name, user.last_name]), user.username)),
to_save=False)
self.save()
def unsubscribe(self, user):
if self.is_subscribed(user):
self.subscribers = [u.id for u in self.subscribers if u != user.id]
self.log_message(
"%s(%s) has been unsubscribed" %
((" ".join([user.first_name, user.last_name]), user.username)),
to_save=False)
self.save()
def is_owner(self, user):
return self.owner == user
def is_subscribed(self, user):
return user.id in self.subscribers
@property
def is_unassigned(self):
return self.owner is None
@property
def duration(self):
dt = datetime.datetime.now() - self.timestamp
return dt.days * 86400 + dt.seconds
@property
def display_duration(self):
duration = datetime.datetime.now() - self.timestamp
secs = duration.seconds % 60
mins = (duration.seconds / 60) % 60
hours = (duration.seconds / 3600) % 24
days = duration.days
r = "%02d:%02d:%02d" % (hours, mins, secs)
if days:
r = "%dd %s" % (days, r)
return r
@property
def effective_style(self):
if self.custom_style:
return self.custom_style
else:
return AlarmSeverity.get_severity(self.severity).style
def set_root(self, root_alarm):
"""
Set root cause
"""
if self.root:
return
if self.id == root_alarm.id:
raise Exception("Cannot set self as root cause")
# Detect loop
root = root_alarm
while root and root.root:
root = root.root
if root == self.id:
return
root = get_alarm(root)
# Set root
self.root = root_alarm.id
self.log_message("Alarm %s has been marked as root cause" %
root_alarm.id)
# self.save() Saved by log_message
root_alarm.log_message("Alarm %s has been marked as child" % self.id)
self._change_root_severity()
# Clear pending notifications
Notification.purge_delayed("alarm:%s" % self.id)
@classmethod
def enable_caching(cls, ttl=600):
cls._fields["alarm_class"].set_cache(ttl)
class AlarmClass(nosql.Document):
"""
Alarm class
"""
meta = {
"collection": "noc.alarmclasses",
"allow_inheritance": False,
"json_collection": "fm.alarmclasses",
"json_depends_on": ["fm.alarmseverities"]
}
name = fields.StringField(required=True, unique=True)
uuid = fields.UUIDField(binary=True)
description = fields.StringField(required=False)
# Create or not create separate Alarm
# if is_unique is True and there is active alarm
# Do not create separate alarm if is_unique set
is_unique = fields.BooleanField(default=False)
# List of var names to be used as discriminator key
discriminator = fields.ListField(nosql.StringField())
# Can alarm status be cleared by user
user_clearable = fields.BooleanField(default=True)
# Default alarm severity
default_severity = nosql.PlainReferenceField(AlarmSeverity)
#
datasources = fields.ListField(fields.EmbeddedDocumentField(DataSource))
vars = fields.ListField(fields.EmbeddedDocumentField(AlarmClassVar))
# Text messages
subject_template = fields.StringField()
body_template = fields.StringField()
symptoms = fields.StringField()
probable_causes = fields.StringField()
recommended_actions = fields.StringField()
# Flap detection
flap_condition = fields.StringField(required=False,
choices=[("none", "none"),
("count", "count")],
default=None)
flap_window = fields.IntField(required=False, default=0)
flap_threshold = fields.FloatField(required=False, default=0)
# RCA
root_cause = fields.ListField(
fields.EmbeddedDocumentField(AlarmRootCauseCondition))
# Job descriptions
jobs = fields.ListField(fields.EmbeddedDocumentField(AlarmClassJob))
#
handlers = fields.ListField(fields.StringField())
# Plugin settings
plugins = fields.ListField(fields.EmbeddedDocumentField(AlarmPlugin))
# Time in seconds to delay alarm risen notification
notification_delay = fields.IntField(required=False)
# Control time to reopen alarm instead of creating new
control_time0 = fields.IntField(required=False)
# Control time to reopen alarm after 1 reopen
control_time1 = fields.IntField(required=False)
# Control time to reopen alarm after >1 reopen
control_timeN = fields.IntField(required=False)
#
category = nosql.ObjectIdField()
def __unicode__(self):
return self.name
def save(self, *args, **kwargs):
c_name = " | ".join(self.name.split(" | ")[:-1])
c = AlarmClassCategory.objects.filter(name=c_name).first()
if not c:
c = AlarmClassCategory(name=c_name)
c.save()
self.category = c.id
super(AlarmClass, self).save(*args, **kwargs)
def get_discriminator(self, vars):
"""
Calculate discriminator hash
:param vars: Dict of vars
:returns: Discriminator hash
"""
if vars:
ds = sorted(str(vars[n]) for n in self.discriminator)
return hashlib.sha1("\x00".join(ds)).hexdigest()
else:
return hashlib.sha1("").hexdigest()
def to_json(self):
c = self
r = ["{"]
r += [" \"name\": \"%s\"," % q(c.name)]
r += [" \"$collection\": \"%s\"," % self._meta["json_collection"]]
r += [" \"uuid\": \"%s\"," % c.uuid]
if c.description:
r += [" \"desciption\": \"%s\"," % q(c.description)]
r += [" \"is_unique\": %s," % q(c.is_unique)]
if c.is_unique and c.discriminator:
r += [
" \"discriminator\": [%s]," %
", ".join(["\"%s\"" % q(d) for d in c.discriminator])
]
r += [" \"user_clearable\": %s," % q(c.user_clearable)]
r += [
" \"default_severity__name\": \"%s\"," %
q(c.default_severity.name)
]
# datasources
if c.datasources:
r += [" \"datasources\": ["]
jds = []
for ds in c.datasources:
x = []
x += [" \"name\": \"%s\"" % q(ds.name)]
x += [" \"datasource\": \"%s\"" % q(ds.datasource)]
ss = []
for k in sorted(ds.search):
ss += [
" \"%s\": \"%s\"" %
(q(k), q(ds.search[k]))
]
x += [
" \"search\": {\n%s\n }" %
(",\n".join(ss))
]
jds += [" {\n%s\n }" % ",\n".join(x)]
r += [",\n\n".join(jds)]
r += [" ],"]
# vars
vars = []
for v in c.vars:
vd = [" {"]
vd += [" \"name\": \"%s\"," % q(v.name)]
vd += [" \"description\": \"%s\"" % q(v.description)]
if v.default:
vd[-1] += ","
vd += [" \"default\": \"%s\"" % q(v.default)]
vd += [" }"]
vars += ["\n".join(vd)]
r += [" \"vars\": ["]
r += [",\n".join(vars)]
r += [" ],"]
# Handlers
if self.handlers:
hh = [" \"%s\"" % h for h in self.handlers]
r += [" \"handlers\": ["]
r += [",\n\n".join(hh)]
r += [" ],"]
# Text
r += [" \"subject_template\": \"%s\"," % q(c.subject_template)]
r += [" \"body_template\": \"%s\"," % q(c.body_template)]
r += [" \"symptoms\": \"%s\"," % q(c.symptoms)]
r += [" \"probable_causes\": \"%s\"," % q(c.probable_causes)]
r += [
" \"recommended_actions\": \"%s\"," % q(c.recommended_actions)
]
# Root cause
if self.root_cause:
rc = []
for rr in self.root_cause:
rcd = [" {"]
rcd += [" \"name\": \"%s\"," % rr.name]
rcd += [" \"root__name\": \"%s\"," % rr.root.name]
rcd += [" \"window\": %d," % rr.window]
if rr.condition:
rcd += [
" \"condition\": \"%s\"," % rr.condition
]
rcd += [" \"match_condition\": {"]
mcv = []
for v in rr.match_condition:
mcv += [
" \"%s\": \"%s\"" %
(v, rr.match_condition[v])
]
rcd += [",\n".join(mcv)]
rcd += [" }"]
rcd += [" }"]
rc += ["\n".join(rcd)]
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"root_cause\": ["]
r += [",\n".join(rc)]
r += [" ]"]
# Jobs
if self.jobs:
jobs = []
for job in self.jobs:
jd = [" {"]
jd += [" \"job\": \"%s\"," % job.job]
jd += [" \"interval\": %d," % job.interval]
jd += [" \"vars\": {"]
jv = []
for v in job.vars:
jv += [" \"%s\": \"%s\"" % (v, job.vars[v])]
jd += [",\n".join(jv)]
jd += [" }"]
jd += [" }"]
jobs += ["\n".join(jd)]
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"jobs\": ["]
r += [",\n".join(jobs)]
r += [" ]"]
# Plugins
if self.plugins:
if r[-1][-1] != ",":
r[-1] += ","
plugins = []
for p in self.plugins:
pd = [" {"]
pd += [" \"name\": \"%s\"" % p.name]
if p.config:
pd[-1] += ","
pc = []
for v in p.config:
pc += [
" \"%s\": \"%s\"" %
(v, p.config.vars[v])
]
pd += [" \"config\": {"]
pd += [",\n".join(pc)]
pd += [" }"]
pd += [" }"]
plugins += ["\n".join(pd)]
r += [" \"plugins\": ["]
r += [",\n".join(plugins)]
r += [" ]"]
if self.notification_delay:
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"notification_delay\": %d" % self.notification_delay]
if self.control_time0:
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"control_time0\": %d" % self.control_time0]
if self.control_time1:
r[-1] += ","
r += [" \"control_time1\": %d" % self.control_time1]
if self.control_timeN:
r[-1] += ","
r += [" \"control_timeN\": %d" % self.control_timeN]
# Close
if r[-1].endswith(","):
r[-1] = r[-1][:-1]
r += ["}", ""]
return "\n".join(r)
def get_json_path(self):
p = [quote_safe_path(n.strip()) for n in self.name.split("|")]
return os.path.join(*p) + ".json"
@property
def config(self):
if not hasattr(self, "_config"):
self._config = AlarmClassConfig.objects.filter(
alarm_class=self.id).first()
return self._config
def get_notification_delay(self):
if self.config:
return self.config.notification_delay or None
else:
return self.notification_delay or None
def get_control_time(self, reopens):
if reopens == 0:
if self.config:
return self.config.control_time0 or None
else:
return self.control_time0 or None
elif reopens == 1:
if self.config:
return self.config.control_time1 or None
else:
return self.control_time1 or None
else:
if self.config:
return self.config.control_timeN or None
else:
return self.control_timeN or None
class AlarmClass(nosql.Document):
"""
Alarm class
"""
meta = {
"collection": "noc.alarmclasses",
"strict": False,
"auto_create_index": False,
"json_collection": "fm.alarmclasses",
"json_depends_on": [
"fm.alarmseverities"
],
}
name = fields.StringField(required=True, unique=True)
uuid = fields.UUIDField(binary=True)
description = fields.StringField(required=False)
# Create or not create separate Alarm
# if is_unique is True and there is active alarm
# Do not create separate alarm if is_unique set
is_unique = fields.BooleanField(default=False)
# List of var names to be used as discriminator key
discriminator = fields.ListField(nosql.StringField())
# Can alarm status be cleared by user
user_clearable = fields.BooleanField(default=True)
# Default alarm severity
default_severity = nosql.PlainReferenceField(AlarmSeverity)
#
datasources = fields.ListField(fields.EmbeddedDocumentField(DataSource))
vars = fields.ListField(fields.EmbeddedDocumentField(AlarmClassVar))
# Text messages
subject_template = fields.StringField()
body_template = fields.StringField()
symptoms = fields.StringField()
probable_causes = fields.StringField()
recommended_actions = fields.StringField()
# Flap detection
flap_condition = fields.StringField(
required=False,
choices=[("none", "none"), ("count", "count")],
default="none")
flap_window = fields.IntField(required=False, default=0)
flap_threshold = fields.FloatField(required=False, default=0)
# RCA
root_cause = fields.ListField(
fields.EmbeddedDocumentField(AlarmRootCauseCondition))
topology_rca = fields.BooleanField(default=False)
# List of handlers to be called on alarm raising
handlers = fields.ListField(fields.StringField())
# List of handlers to be called on alarm clear
clear_handlers = fields.ListField(fields.StringField())
# Plugin settings
plugins = fields.ListField(fields.EmbeddedDocumentField(AlarmPlugin))
# Time in seconds to delay alarm risen notification
notification_delay = fields.IntField(required=False)
# Control time to reopen alarm instead of creating new
control_time0 = fields.IntField(required=False)
# Control time to reopen alarm after 1 reopen
control_time1 = fields.IntField(required=False)
# Control time to reopen alarm after >1 reopen
control_timeN = fields.IntField(required=False)
# Consequence recover time
# Root cause will be detached if consequence alarm
# will not clear itself in *recover_time*
recover_time = fields.IntField(required=False, default=300)
#
bi_id = fields.LongField(unique=True)
#
category = nosql.ObjectIdField()
_id_cache = cachetools.TTLCache(maxsize=1000, ttl=60)
_bi_id_cache = cachetools.TTLCache(maxsize=1000, ttl=60)
_name_cache = cachetools.TTLCache(maxsize=1000, ttl=60)
_handlers_cache = {}
_clear_handlers_cache = {}
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
def get_by_id(cls, id):
return AlarmClass.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock)
def get_by_bi_id(cls, id):
return AlarmClass.objects.filter(bi_id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_name_cache"), lock=lambda _: id_lock)
def get_by_name(cls, name):
return AlarmClass.objects.filter(name=name).first()
def get_handlers(self):
@cachetools.cached(self._handlers_cache, key=lambda x: x.id, lock=handlers_lock)
def _get_handlers(alarm_class):
handlers = []
for hh in alarm_class.handlers:
try:
h = get_handler(hh)
except ImportError:
h = None
if h:
handlers += [h]
return handlers
return _get_handlers(self)
def get_clear_handlers(self):
@cachetools.cached(self._clear_handlers_cache, key=lambda x: x.id, lock=handlers_lock)
def _get_handlers(alarm_class):
handlers = []
for hh in alarm_class.clear_handlers:
try:
h = get_handler(hh)
except ImportError:
h = None
if h:
handlers += [h]
return handlers
return _get_handlers(self)
def save(self, *args, **kwargs):
c_name = " | ".join(self.name.split(" | ")[:-1])
c = AlarmClassCategory.objects.filter(name=c_name).first()
if not c:
c = AlarmClassCategory(name=c_name)
c.save()
self.category = c.id
super(AlarmClass, self).save(*args, **kwargs)
def get_discriminator(self, vars):
"""
Calculate discriminator hash
:param vars: Dict of vars
:returns: Discriminator hash
"""
if vars:
ds = sorted(str(vars[n]) for n in self.discriminator)
return hashlib.sha1("\x00".join(ds)).hexdigest()
else:
return hashlib.sha1("").hexdigest()
def to_json(self):
c = self
r = ["{"]
r += [" \"name\": \"%s\"," % q(c.name)]
r += [" \"$collection\": \"%s\"," % self._meta["json_collection"]]
r += [" \"uuid\": \"%s\"," % c.uuid]
if c.description:
r += [" \"desciption\": \"%s\"," % q(c.description)]
r += [" \"is_unique\": %s," % q(c.is_unique)]
if c.is_unique and c.discriminator:
r += [" \"discriminator\": [%s]," % ", ".join(["\"%s\"" % q(d) for d in c.discriminator])]
r += [" \"user_clearable\": %s," % q(c.user_clearable)]
r += [" \"default_severity__name\": \"%s\"," % q(c.default_severity.name)]
# datasources
if c.datasources:
r += [" \"datasources\": ["]
jds = []
for ds in c.datasources:
x = []
x += [" \"name\": \"%s\"" % q(ds.name)]
x += [" \"datasource\": \"%s\"" % q(ds.datasource)]
ss = []
for k in sorted(ds.search):
ss += [" \"%s\": \"%s\"" % (q(k), q(ds.search[k]))]
x += [" \"search\": {\n%s\n }" % (",\n".join(ss))]
jds += [" {\n%s\n }" % ",\n".join(x)]
r += [",\n\n".join(jds)]
r += [" ],"]
# vars
vars = []
for v in c.vars:
vd = [" {"]
vd += [" \"name\": \"%s\"," % q(v.name)]
vd += [" \"description\": \"%s\"" % q(v.description)]
if v.default:
vd[-1] += ","
vd += [" \"default\": \"%s\"" % q(v.default)]
vd += [" }"]
vars += ["\n".join(vd)]
r += [" \"vars\": ["]
r += [",\n".join(vars)]
r += [" ],"]
# Handlers
if self.handlers:
hh = [" \"%s\"" % h for h in self.handlers]
r += [" \"handlers\": ["]
r += [",\n\n".join(hh)]
r += [" ],"]
if self.clear_handlers:
hh = [" \"%s\"" % h for h in self.clear_handlers]
r += [" \"clear_handlers\": ["]
r += [",\n\n".join(hh)]
r += [" ],"]
# Text
r += [" \"subject_template\": \"%s\"," % q(c.subject_template)]
r += [" \"body_template\": \"%s\"," % q(c.body_template)]
r += [" \"symptoms\": \"%s\"," % q(c.symptoms if c.symptoms else "")]
r += [" \"probable_causes\": \"%s\"," % q(c.probable_causes if c.probable_causes else "")]
r += [" \"recommended_actions\": \"%s\"," % q(c.recommended_actions if c.recommended_actions else "")]
# Root cause
if self.root_cause:
rc = []
for rr in self.root_cause:
rcd = [" {"]
rcd += [" \"name\": \"%s\"," % rr.name]
rcd += [" \"root__name\": \"%s\"," % rr.root.name]
rcd += [" \"window\": %d," % rr.window]
if rr.condition:
rcd += [" \"condition\": \"%s\"," % rr.condition]
rcd += [" \"match_condition\": {"]
mcv = []
for v in rr.match_condition:
mcv += [" \"%s\": \"%s\"" % (v, rr.match_condition[v])]
rcd += [",\n".join(mcv)]
rcd += [" }"]
rcd += [" }"]
rc += ["\n".join(rcd)]
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"root_cause\": ["]
r += [",\n".join(rc)]
r += [" ]"]
if self.topology_rca:
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"topology_rca\": true"]
# Plugins
if self.plugins:
if r[-1][-1] != ",":
r[-1] += ","
plugins = []
for p in self.plugins:
pd = [" {"]
pd += [" \"name\": \"%s\"" % p.name]
if p.config:
pd[-1] += ","
pc = []
for v in p.config:
pc += [" \"%s\": \"%s\"" % (v, p.config.vars[v])]
pd += [" \"config\": {"]
pd += [",\n".join(pc)]
pd += [" }"]
pd += [" }"]
plugins += ["\n".join(pd)]
r += [" \"plugins\": ["]
r += [",\n".join(plugins)]
r += [" ]"]
if self.notification_delay:
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"notification_delay\": %d" % self.notification_delay]
if self.control_time0:
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"control_time0\": %d" % self.control_time0]
if self.control_time1:
r[-1] += ","
r += [" \"control_time1\": %d" % self.control_time1]
if self.control_timeN:
r[-1] += ","
r += [" \"control_timeN\": %d" % self.control_timeN]
if self.recover_time:
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"recover_time\": %d" % self.recover_time]
# Close
if r[-1].endswith(","):
r[-1] = r[-1][:-1]
r += ["}", ""]
return "\n".join(r)
def get_json_path(self):
p = [quote_safe_path(n.strip()) for n in self.name.split("|")]
return os.path.join(*p) + ".json"
@property
def config(self):
if not hasattr(self, "_config"):
self._config = AlarmClassConfig.objects.filter(alarm_class=self.id).first()
return self._config
def get_notification_delay(self):
if self.config:
return self.config.notification_delay or None
else:
return self.notification_delay or None
def get_control_time(self, reopens):
if reopens == 0:
if self.config:
return self.config.control_time0 or None
else:
return self.control_time0 or None
elif reopens == 1:
if self.config:
return self.config.control_time1 or None
else:
return self.control_time1 or None
else:
if self.config:
return self.config.control_timeN or None
else:
return self.control_timeN or None
class ArchivedAlarm(nosql.Document):
meta = {
"collection": "noc.alarms.archived",
"allow_inheritance": False,
"indexes": [
"root",
"control_time",
"timestamp",
"managed_object"
]
}
status = "C"
timestamp = nosql.DateTimeField(required=True)
clear_timestamp = nosql.DateTimeField(required=True)
managed_object = nosql.ForeignKeyField(ManagedObject)
alarm_class = nosql.PlainReferenceField(AlarmClass)
severity = nosql.IntField(required=True)
vars = nosql.DictField()
log = nosql.ListField(nosql.EmbeddedDocumentField(AlarmLog))
#
opening_event = nosql.ObjectIdField(required=False)
closing_event = nosql.ObjectIdField(required=False)
# Number of reopens
reopens = nosql.IntField(required=False)
# Copied discriminator
discriminator = nosql.StringField(required=False)
# Control time within alarm will be reopen instead
# instead of creating the new alarm
control_time = nosql.DateTimeField(required=False)
# RCA
# Reference to root cause (Active Alarm or Archived Alarm instance)
root = nosql.ObjectIdField(required=False)
def __unicode__(self):
return u"%s" % self.id
def log_message(self, message):
self.log += [AlarmLog(timestamp=datetime.datetime.now(),
from_status=self.status, to_status=self.status,
message=message)]
self.save()
def get_template_vars(self):
"""
Prepare template variables
"""
vars = self.vars.copy()
vars.update({"event": self})
return vars
@property
def subject(self):
ctx = Context(self.get_template_vars())
s = Template(self.alarm_class.subject_template).render(ctx)
if len(s) >= 255:
s = s[:125] + " ... " + s[-125:]
return s
@property
def body(self):
ctx = Context(self.get_template_vars())
s = Template(self.alarm_class.body_template).render(ctx)
return s
@property
def duration(self):
dt = self.clear_timestamp - self.timestamp
return dt.days * 86400 + dt.seconds
@property
def display_duration(self):
duration = self.clear_timestamp - self.timestamp
secs = duration.seconds % 60
mins = (duration.seconds / 60) % 60
hours = (duration.seconds / 3600) % 24
days = duration.days
if days:
return "%dd %02d:%02d:%02d" % (days, hours, mins, secs)
else:
return "%02d:%02d:%02d" % (hours, mins, secs)
@property
def effective_style(self):
return AlarmSeverity.get_severity(self.severity).style
def set_root(self, root_alarm):
pass
def reopen(self, message):
"""
Reopen alarm back
"""
reopens = self.reopens or 0
ts = datetime.datetime.now()
log = self.log + [AlarmLog(timestamp=ts, from_status="C",
to_status="A", message=message)]
a = ActiveAlarm(
id=self.id,
timestamp=self.timestamp,
last_update=ts,
managed_object=self.managed_object,
alarm_class=self.alarm_class,
severity=self.severity,
vars=self.vars,
log=log,
root=self.root,
opening_event=self.opening_event,
discriminator=self.discriminator,
reopens=reopens + 1
)
a.save()
# @todo: Clear related correlator jobs
self.delete()
# Remove pending control_notify job
remove_job("fm.correlator", "control_notify", key=a.id)
# Send notifications
# Do not set notifications for child and for previously reopened
# alarms
if not a.root and not reopens:
a.managed_object.event(a.managed_object.EV_ALARM_REOPENED, {
"alarm": a,
"subject": a.subject,
"body": a.body,
"symptoms": a.alarm_class.symptoms,
"recommended_actions": a.alarm_class.recommended_actions,
"probable_causes": a.alarm_class.probable_causes
})
return a
class MIB(nosql.Document):
meta = {"collection": "noc.mibs", "allow_inheritance": False}
name = nosql.StringField(required=True, unique=True)
description = nosql.StringField(required=False)
last_updated = nosql.DateTimeField(required=True)
depends_on = nosql.ListField(nosql.StringField())
# TC definitions: name -> SYNTAX
typedefs = nosql.DictField(required=False)
# Compiled MIB version
version = nosql.IntField(required=False, default=0)
MIBRequiredException = MIBRequiredException
def __unicode__(self):
return self.name
def get_text(self):
"""
Returns MIB text
:return:
"""
for d in ["local/share/mibs", "share/mibs"]:
path = os.path.join(d, self.name + ".mib")
if os.path.isfile(path):
with open(path) as f:
return f.read()
return ""
@classmethod
def parse_syntax(cls, syntax):
"""
Process part of smidump output and convert to syntax structure
"""
if "base_type" in syntax:
# Already compiled
return syntax
s = {}
if "basetype" in syntax:
s["base_type"] = syntax["basetype"]
elif "base_type" in syntax:
s["base_type"] = syntax["base_type"]
if "name" in syntax and "module" in syntax:
if syntax["module"] == "":
# Empty module -> builitin types
s["base_type"] = syntax["name"]
else:
# Resolve references
mib = MIB.objects.filter(name=syntax["module"]).first()
if mib is None:
raise MIBNotFoundException(syntax["module"])
if not mib.typedefs or syntax["name"] not in mib.typedefs:
return {}
td = mib.typedefs[syntax["name"]]
for k in ["base_type", "display_hint", "enum_map"]:
if k in td:
s[k] = td[k]
if s["base_type"] in ("Enumeration", "Bits"):
enum_map = s.get("enum_map", {})
for k in syntax:
sk = syntax[k]
if type(sk) != dict:
continue
if "nodetype" in sk and sk["nodetype"] == "namednumber":
enum_map[sk["number"]] = k
s["enum_map"] = enum_map
if "format" in syntax:
s["display_hint"] = syntax["format"]
return s
@classmethod
def load(cls, path, force=False):
"""
Load MIB from file
:param path: MIB path
:param force: Load anyways
:return: MIB object
"""
if not os.path.exists(path):
raise ValueError("File not found: %s" % path)
# Build SMIPATH variable for smidump
# to exclude locally installed MIBs
smipath = ["share/mibs", "local/share/mibs"]
# Pass MIB through smilint to detect missed modules
f = subprocess.Popen([config.get("path", "smilint"), "-m", path],
stderr=subprocess.PIPE,
env={
"SMIPATH": ":".join(smipath)
}).stderr
for l in f:
match = rx_module_not_found.search(l.strip())
if match:
raise MIBRequiredException("Uploaded MIB", match.group(1))
# Convert MIB to python module and load
with temporary_file() as p:
subprocess.check_call([
config.get("path", "smidump"), "-k", "-q", "-f", "python",
"-o", p, path
],
env={"SMIPATH": ":".join(smipath)})
# Add coding string
with open(p) as f:
data = unicode(f.read(), "ascii", "ignore").encode("ascii")
with open(p, "w") as f:
f.write(data)
m = imp.load_source("mib", p)
mib_name = m.MIB["moduleName"]
# Check module dependencies
depends_on = {} # MIB Name -> Object ID
if "imports" in m.MIB:
for i in m.MIB["imports"]:
if "module" not in i:
continue
rm = i["module"]
if rm in depends_on:
continue
md = MIB.objects.filter(name=rm).first()
if md is None:
raise MIBRequiredException(mib_name, rm)
depends_on[rm] = md
# Get MIB latest revision date
try:
last_updated = datetime.datetime.strptime(
sorted([x["date"] for x in m.MIB[mib_name]["revisions"]])[-1],
"%Y-%m-%d %H:%M")
except:
last_updated = datetime.datetime(year=1970, month=1, day=1)
# Extract MIB typedefs
typedefs = {}
if "typedefs" in m.MIB:
for t in m.MIB["typedefs"]:
typedefs[t] = cls.parse_syntax(m.MIB["typedefs"][t])
# Check mib already uploaded
mib_description = m.MIB[mib_name].get("description", None)
mib = MIB.objects.filter(name=mib_name).first()
if force and mib:
# Delete mib to forceful update
MIBData.objects.filter(mib=mib.id).delete()
mib.clean()
mib.delete()
mib = None
if mib is not None:
# Skip same version
if mib.last_updated >= last_updated:
return mib
mib.description = mib_description
mib.last_updated = last_updated
mib.depends_on = sorted(depends_on)
mib.typedefs = typedefs
mib.save()
# Delete all MIB Data
mib.clean()
else:
# Create MIB
mib = MIB(name=mib_name,
description=mib_description,
last_updated=last_updated,
depends_on=sorted(depends_on),
typedefs=typedefs)
mib.save()
# Upload MIB data
data = []
for i in ["nodes", "notifications"]:
if i in m.MIB:
data += [{
"name":
"%s::%s" % (mib_name, node),
"oid":
v["oid"],
"description":
v.get("description"),
"syntax":
v["syntax"]["type"] if "syntax" in v else None
} for node, v in m.MIB[i].items()]
mib.load_data(data)
# Save MIB to cache if not uploaded from cache
lcd = os.path.join("local", "share", "mibs")
if not os.path.isdir(lcd): # Ensure directory exists
os.makedirs(os.path.join("local", "share", "mibs"))
local_cache_path = os.path.join(lcd, "%s.mib" % mib_name)
cache_path = os.path.join("share", "mibs", "%s.mib" % mib_name)
if ((os.path.exists(local_cache_path)
and os.path.samefile(path, local_cache_path))
or (os.path.exists(cache_path)
and os.path.samefile(path, cache_path))):
return mib
with open(path) as f:
data = f.read()
safe_rewrite(local_cache_path, data)
return mib
def load_data(self, data):
"""
Load mib data from list of {oid:, name:, description:, syntax:}
:param data:
:return:
"""
# Get MIB preference
mp = MIBPreference.objects.filter(mib=self.name).first()
mib_preference = mp.preference if mp else None
prefs = {} # MIB Preferences cache
# Load data
for v in data:
oid = v["oid"]
oid_name = v["name"]
description = v.get("description", None)
o = MIBData.objects.filter(oid=oid).first()
if o is not None:
if o.name == oid_name:
# Same oid, same name: duplicated declaration.
# Silently skip
continue
# For same MIB - leave first entry
if oid_name.split("::", 1)[0] == o.name.split("::", 1)[0]:
continue
# Try to resolve collision
if not mib_preference:
# No preference for target MIB
raise OIDCollision(oid, oid_name, o.name,
"No preference for %s" % self.name)
o_mib = o.name.split("::")[0]
if o_mib not in prefs:
mp = MIBPreference.objects.filter(mib=o_mib).first()
if not mp:
# No preference for destination MIB
raise OIDCollision(oid, oid_name, o.name,
"No preference for %s" % o_mib)
prefs[o_mib] = mp.preference # Add to cache
o_preference = prefs[o_mib]
if mib_preference == o_preference:
# Equal preferences, collision
raise OIDCollision(oid, oid_name, o.name,
"Equal preferences")
if mib_preference < o_preference:
# Replace existing
o.aliases = sorted(o.aliases + [o.name])
o.name = oid_name
o.mib = self.id
if description:
o.description = description
syntax = v.get("syntax")
if syntax:
o.syntax = MIB.parse_syntax(syntax)
o.save()
else:
# Append to aliases
if oid_name not in o.aliases:
o.aliases = sorted(o.aliases + [oid_name])
o.save()
else:
# No OID collision found, save
syntax = v.get("syntax")
if syntax:
syntax = MIB.parse_syntax(syntax)
MIBData(mib=self.id,
oid=oid,
name=oid_name,
description=description,
syntax=syntax).save()
@classmethod
def get_oid(cls, name):
"""
Get OID by name
"""
tail = ""
match = rx_tailing_numbers.match(name)
if match:
name, tail = match.groups()
# Search by primary name
d = MIBData.objects.filter(name=name).first()
if not d:
# Search by aliases
d = MIBData.objects.filter(aliases=name).first()
if d:
return d.oid + tail
return None
@classmethod
def get_name(cls, oid):
"""
Get longest match name by OID
"""
oid = OIDAlias.rewrite(oid)
l_oid = oid.split(".")
rest = []
while l_oid:
c_oid = ".".join(l_oid)
d = MIBData.objects.filter(oid=c_oid).first()
if d:
return MIBAlias.rewrite(".".join([d.name] + rest))
else:
rest = [l_oid.pop()] + rest
return oid
@classmethod
def get_name_and_syntax(cls, oid):
"""
:return: (name, syntax)
"""
oid = OIDAlias.rewrite(oid)
l_oid = oid.split(".")
rest = []
while l_oid:
c_oid = ".".join(l_oid)
d = MIBData.objects.filter(oid=c_oid).first()
if d:
name = d.name
if rest:
name += "." + ".".join(reversed(rest))
return (MIBAlias.rewrite(name),
SyntaxAlias.rewrite(name, d.syntax))
else:
rest += [l_oid.pop()]
return oid, None
@classmethod
def get_description(cls, name):
"""
Get longest match description by name
"""
match = rx_tailing_numbers.match(name)
if match:
name, _ = match.groups()
# Search by primary name
d = MIBData.objects.filter(name=name).first()
if not d:
# Search by aliases
d = MIBData.objects.filter(aliases=name).first()
if d:
return d.description
else:
return None
@property
def depended_by(self):
return MIB.objects.filter(depends_on=self.name)
def clean(self):
"""
Gracefully wipe out MIB data
"""
# Delete data without aliases
MIBData.objects.filter(mib=self.id, aliases=[]).delete()
# Dereference aliases
prefs = {} # MIB -> Preference
for o in MIBData.objects.filter(mib=self.id, aliases__ne=[]):
if not o.aliases: # NO aliases
o.delete()
continue
if len(o.aliases) == 1: # Only one alias
ba = o.aliases[0]
else:
# Find preferable alias
ba = None
lp = None
for a in o.aliases:
am = a.split("::")[0]
# Find MIB preference
if am not in prefs:
p = MIBPreference(mib=am).first()
if p is None:
raise Exception("No preference for %s" % am)
prefs[am] = p.preference
p = prefs[am]
if lp is None or p < lp:
# Better
ba = a
lp = p
# Promote preferable alias
o.name = ba
o.aliases = [a for a in o.aliases if a != ba]
o.save()
@classmethod
def resolve_vars(cls, vars):
"""
Resolve FM key -> value dict according to MIBs
:param cls:
:param vars:
:return:
"""
r = {}
for k in vars:
if not is_oid(k):
# Nothing to resolve
continue
v = fm_unescape(vars[k])
rk, syntax = cls.get_name_and_syntax(k)
rv = v
if syntax:
# Format value according to syntax
if syntax["base_type"] == "Enumeration":
# Expand enumerated type
try:
rv = syntax["enum_map"][str(v)]
except KeyError:
pass
elif syntax["base_type"] == "Bits":
# @todo: Fix ugly hack
if v.startswith("="):
xv = int(v[1:], 16)
else:
xv = 0
for c in v:
xv = (xv << 8) + ord(c)
# Decode
b_map = syntax.get("enum_map", {})
b = []
n = 0
while xv:
if xv & 1:
x = str(n)
if x in b_map:
b = [b_map[x]] + b
else:
b = ["%X" % (1 << n)]
n += 1
xv >>= 1
rv = "(%s)" % ",".join(b)
else:
# Render according to TC
rv = render_tc(v, syntax["base_type"],
syntax.get("display_hint", None))
try:
unicode(rv, "utf8")
except:
# Escape invalid UTF8
rv = fm_escape(rv)
else:
try:
unicode(rv, "utf8")
except:
# escape invalid UTF8
rv = fm_escape(rv)
if is_oid(v):
# Resolve OID in value
rv = MIB.get_name(v)
if rk != k or rv != v:
r[rk] = rv
return r
