class ForwardingInstance(Document):
"""
Non-default forwarding instances
"""
meta = {
"collection": "noc.forwardinginstances",
"allow_inheritance": False,
"indexes": ["managed_object"]
}
managed_object = ForeignKeyField(ManagedObject)
type = StringField(choices=[
(x, x) for x in ("ip", "bridge", "VRF", "VPLS", "VLL")
],
default="ip")
virtual_router = StringField(required=False)
name = StringField()
# VRF/VPLS
rd = StringField(required=False)
def __unicode__(self):
return u"%s: %s" % (self.managed_object.name,
self.name if self.name else "default")
def delete(self, *args, **kwargs):
# Delete subinterfaces
for si in self.subinterface_set.all():
si.delete()
# Delete forwarding instance
super(ForwardingInstance, self).delete(*args, **kwargs)
@property
def subinterface_set(self):
return SubInterface.objects.filter(forwarding_instance=self.id)
class MACRange(EmbeddedDocument):
meta = {
"allow_inheritance": False
}
first_mac = StringField()
last_mac = StringField()
def __unicode__(self):
return u"%s - %s" % (self.first_mac, self.last_mac)
class MetricSet(Document):
meta = {"collection": "noc.pm.metricsets"}
name = StringField(unique=True)
is_active = BooleanField(default=True)
description = StringField(required=False)
interval = IntField(default=60)
metrics = ListField(EmbeddedDocumentField(MetricItem))
def __unicode__(self):
return self.name
def get_effective_metrics(self):
"""
Returns a list of MetricItems, containing all effective
metrics and thresholds for group
"""
def apply_settings(name, mi):
"""
Apply settings to node and all children
"""
dst = mt_tree[name][0]
dst.is_active = mi.is_active
dst.low_warn = mi.low_warn
dst.high_warn = mi.high_warn
dst.low_error = mi.low_error
dst.high_error = mi.high_error
for c in mt_tree[name][1]:
apply_settings(c, mi)
# Build metric type tree
mt_tree = {} # Metric type name -> (metric item, [children])
for mi in self.metrics:
mt = mi.metric_type
if mt.name in mt_tree:
continue
# Find all children
nmt = [mt] + sorted(
MetricType.objects.filter(name__startswith=mt.name + " | "),
key=lambda x: len(x.name))
for m in nmt:
if m.name in mt_tree:
continue
mt_tree[m.name] = [
MetricItem(metric_type=m, is_active=True), []
]
parent = " | ".join(p for p in m.name.split(" | ")[:-1])
if parent in mt_tree:
mt_tree[parent][1] += [m.name]
# Apply settings
for mi in self.metrics:
apply_settings(mi.metric_type.name, mi)
# Fetch leaf nodes
r = [mi[0] for mi in mt_tree.itervalues() if not mi[1]]
return r
class MRTConfig(Document):
meta = {"collection": "noc.mrtconfig", "allow_inheritance": False}
name = StringField(unique=True)
is_active = BooleanField(default=True)
description = StringField(required=False)
permission_name = StringField(required=True)
selector = ForeignKeyField(ManagedObjectSelector, required=True)
reduce_pyrule = ForeignKeyField(PyRule, required=True)
map_script = StringField(required=True)
timeout = IntField(required=False)
def __unicode__(self):
return self.name
class Tag(Document):
meta = {
"collection": "noc.tags",
"allow_inheritance": False,
"indexes": ["models"]
}
tag = StringField(unique=True)
models = ListField(StringField())
count = IntField(default=0)
def __unicode__(self):
return self.tag
@classmethod
def register_tag(cls, tag, model):
"""
Register new tag occurence
:param model:
:return:
"""
cls._get_collection().update({"tag": tag}, {
"$addToSet": {
"models": model
},
"$inc": {
"count": 1
}
},
upsert=True)
@classmethod
def unregister_tag(cls, tag, model):
"""
Unregister tag occurence
:param model:
:return:
"""
pass
def get_objects(self):
"""
Return all tagged objects
:return:
"""
r = []
for m in self.models:
al, mn = m.split("_", 1)
mc = ContentType.objects.get(app_label=al, model=mn)
r += [mc.objects.filter(QTags([self.tag]))]
return r
class Tag(Document):
meta = {
"collection": "noc.tags",
"strict": False,
"auto_create_index": False,
"indexes": ["models"]
}
tag = StringField(unique=True)
models = ListField(StringField())
count = IntField(default=0)
def __unicode__(self):
return self.tag
@classmethod
def register_tag(cls, tag, model):
"""
Register new tag occurence
:param tag: Tag Name
:param model: Model for creating tag
:return:
"""
cls._get_collection().update_one({"tag": tag}, {
"$addToSet": {
"models": model
},
"$inc": {
"count": 1
}
},
upsert=True)
@classmethod
def unregister_tag(cls, tag, model):
"""
Unregister tag occurence
:param tag: Tag Name
:param model: Model for creating tag
:return:
"""
cls._get_collection().update_one({"tag": tag}, {
"$addToSet": {
"models": model
},
"$inc": {
"count": -1
}
},
upsert=True)
class L3Link(Document):
"""
Network L3 links.
Always contains a list of subinterface references
"""
meta = {
"collection": "noc.links",
"strict": False,
"auto_create_index": False,
"indexes": ["subinterfaces", "linked_objects"]
}
subinterfaces = PlainReferenceListField("inv.SubInterface")
# List of linked objects
linked_objects = ListField(IntField())
# Name of discovery method or "manual"
discovery_method = StringField()
# Timestamp of first discovery
first_discovered = DateTimeField(default=datetime.datetime.now)
# Timestamp of last confirmation
last_seen = DateTimeField()
# L3 path cost
l3_cost = IntField(default=1)
def __unicode__(self):
return u"(%s)" % ", ".join([unicode(i) for i in self.subinterfaces])
def clean(self):
self.linked_objects = sorted(
set(i.managed_object.id for i in self.subinterfaces))
super(L3Link, self).clean()
class MACLog(Document):
"""
Customer MAC address changes
"""
meta = {
"collection": "noc.mac_log",
"allow_inheritance": False,
"indexes": ["mac", "-timestamp"]
}
# Todo: Add Validation
timestamp = DateTimeField()
mac = StringField()
vc_domain_name = StringField()
vlan = IntField()
managed_object_name = StringField()
interface_name = StringField()
class PendingLinkCheck(Document):
"""
Customer MAC address changes
"""
meta = {
"collection": "noc.inv.pending_link_check",
"allow_inheritance": False,
"indexes": [("method", "local_object")]
}
method = StringField()
local_object = ForeignKeyField(ManagedObject)
local_interface = StringField() # optional
remote_object = ForeignKeyField(ManagedObject)
remote_interface = StringField()
expire = DateTimeField()
def __unicode__(self):
return u"%s:%s:%s:%s:%s" % (
self.method, self.local_object.name, self.local_interface,
self.remote_object.name, self.remote_interface)
@classmethod
def submit(cls, method, local_object, local_interface, remote_object,
remote_interface):
expire = datetime.datetime.now() + datetime.timedelta(days=2)
plc = PendingLinkCheck.objects.filter(
method=method,
local_object=local_object.id,
local_interface=local_interface,
remote_object=remote_object.id,
remote_interface=remote_interface).first()
if plc:
plc.expire = expire
else:
plc = cls(method=method,
local_object=local_object.id,
local_interface=local_interface,
remote_object=remote_object.id,
remote_interface=remote_interface,
expire=expire)
plc.save()
class NetworkChartState(Document):
"""
Network Chart State
"""
meta = {
"collection": "noc.inv.networkchartstate",
"allow_inheritance": False,
"indexes": [("chart", "type", "object")]
}
chart = IntField() # Network chart reference
type = StringField(
choices=[
("mo", "Managed Object"),
("link", "Link")
])
object = StringField() # Object reference
state = DictField() # Arbitrary state data
def __unicode__(self):
return "%s %s %s" % (self.chart, self.type, self.object)
class FailedScriptLog(Document):
meta = {
"collection": "noc.log.sa.failed_scripts",
"allow_inheritance": False,
"indexes":
["-timestamp", {
"fields": ["expires"],
"expireAfterSeconds": 0
}]
}
timestamp = DateTimeField()
managed_object = StringField()
address = StringField()
script = StringField()
error_code = IntField()
error_text = StringField()
expires = DateTimeField()
def __unicode__(self):
return str(self.id)
class InterfaceProfile(Document):
"""
Interface SLA profile and settings
"""
meta = {"collection": "noc.interface_profiles", "allow_inheritance": False}
name = StringField(unique=True)
description = StringField()
style = ForeignKeyField(Style, required=False)
# Interface-level events processing
link_events = StringField(required=True,
choices=[("I", "Ignore Events"),
("L",
"Log events, do not raise alarms"),
("A", "Raise alarms")],
default="A")
# Discovery settings
mac_discovery = BooleanField(default=False)
# check_link alarm job interval settings
# Either None or T0,I0,T1,I1,...,Tn-1,In-1,,In
# See MultiIntervalJob settings for details
check_link_interval = StringField(default=",60")
def __unicode__(self):
return self.name
@classmethod
def get_default_profile(cls):
try:
return cls._default_profile
except AttributeError:
cls._default_profile = cls.objects.filter(name="default").first()
return cls._default_profile
def get_probe_config(self, config):
try:
return get_probe_config(self, config)
except ValueError:
pass
raise ValueError("Invalid config '%s'" % config)
class Discovery(Document):
meta = {
"collection": "noc.schedules.inv.discovery",
"strict": False,
"auto_create_index": False
}
job_class = StringField(db_field='jcls')
schedule = DictField()
ts = DateTimeField(db_field='ts')
last = DateTimeField()
last_success = DateTimeField(db_field='st')
last_duration = FloatField(db_field='ldur')
last_status = StringField(db_field='ls')
status = StringField(db_field='s')
managed_object = ForeignKeyField(ManagedObject, db_field='key')
data = DictField()
traceback = DictField()
runs = IntField()
faults = IntField(db_field='f')
log = ListField()
def __unicode__(self):
return "%s: %s" % (self.managed_object, self.job_class)
class ObjectMap(Document):
meta = {
"collection": "noc.cache.object_map",
"allow_inheritance": False,
"indexes": ["object", "collector"]
}
# Object id
object = IntField(required=True, unique=True)
#
collector = IntField(required=True)
#
sources = ListField(StringField())
def __unicode__(self):
return u"%s: %s" % (self.object, self.sources)
@classmethod
def update_map(cls, object, collector, sources):
if hasattr(object, "id"):
object = object.id
if hasattr(collector, "id"):
collector = collector.id
if not isinstance(sources, (list, tuple)):
sources = [sources]
cls._get_collection().update(
{"object": object},
{"$set": {
"collector": collector,
"sources": sources
}},
upsert=True)
@classmethod
def delete_map(cls, object):
if hasattr(object, "id"):
object = object.id
cls._get_collection().remove({"object": object})
@classmethod
def get_map(cls, collector):
c = cls._get_collection()
return list(
c.find({"collector": collector}, {
"object": 1,
"sources": 1,
"_id": 0
}))
class Cache(Document):
meta = {
"collection": "noc.cache",
"allow_inheritance": False,
"indexes": [
{
"fields": ["expires"],
"expireAfterSeconds": 0
}
]
}
key = StringField(db_field="_id", primary_key=True)
value = BinaryField(db_field="v")
pickled_value = BinaryField(db_field="p")
expires = DateTimeField(db_field="e")
def __unicode__(self):
return unicode(self.key)
class Favorites(Document):
meta = {
"collection": "noc.favorites",
"strict": False,
"auto_create_index": False,
"indexes": ["user", ("user", "app")]
}
user = ForeignKeyField(User)
app = StringField()
favorite_app = BooleanField(default=False)
favorites = ListField()
def unicode(self):
return "%s:%s" % (self.user.username, self.app)
@classmethod
def add_item(cls, user, app_id, item):
fv = Favorites.objects.filter(
user=user.id, app=app_id).first()
if not fv:
fv = Favorites(user=user.id, app=app_id, favorites=[])
fi = list(fv.favorites) or []
if item not in fi:
logger.info("Setting favorite item %s@%s for user %s",
item, app_id, user.username)
fv.favorites = fi + [item]
fv.save()
@classmethod
def remove_item(cls, user, app_id, item):
fv = Favorites.objects.filter(
user=user.id, app=app_id).first()
fi = list(fv.favorites) or []
if fv and item and item in fi:
logger.info("Resetting favorite item %s@%s for user %s",
item, app_id, user.username)
fi.remove(item)
fv.favorites = fi
fv.save()
class CollectionCache(Document):
meta = {
"collection": "noc.collectioncache",
"allow_inheritance": False,
"indexes": ["collection"]
}
collection = StringField()
uuid = UUIDField(unique=True, binary=True)
def unicode(self):
return "%s:%s" % (self.collection, self.uuid)
@classmethod
def merge(cls, collection, uuids):
"""
Merge UUIDs to cache
"""
current = set(
o.uuid
for o in CollectionCache.objects.filter(collection=collection))
for u in uuids - current:
CollectionCache(collection=collection, uuid=u).save()
class SubInterface(Document):
meta = {
"collection":
"noc.subinterfaces",
"strict":
False,
"auto_create_index":
False,
"indexes": [("managed_object", "ifindex"),
("managed_object", "vlan_ids"), "interface",
"managed_object", "untagged_vlan", "tagged_vlans",
"enabled_afi", {
"fields": ["ipv4_addresses"],
"sparse": True
}]
}
interface = PlainReferenceField(Interface)
managed_object = ForeignKeyField(ManagedObject)
forwarding_instance = PlainReferenceField(ForwardingInstance,
required=False)
name = StringField()
description = StringField(required=False)
profile = PlainReferenceField(InterfaceProfile,
default=InterfaceProfile.get_default_profile)
mtu = IntField(required=False)
mac = StringField(required=False)
vlan_ids = ListField(IntField(), default=[])
enabled_afi = ListField(StringField(choices=[(x, x)
for x in SUBINTERFACE_AFI]),
default=[])
ipv4_addresses = ListField(StringField(), default=[])
ipv6_addresses = ListField(StringField(), default=[])
iso_addresses = ListField(StringField(), default=[])
vpi = IntField(required=False)
vci = IntField(required=False)
enabled_protocols = ListField(
StringField(choices=[(x, x) for x in SUBINTERFACE_PROTOCOLS]),
default=[])
untagged_vlan = IntField(required=False)
tagged_vlans = ListField(IntField(), default=[])
# ip_unnumbered_subinterface
ifindex = IntField(required=False)
# Tunnel services
tunnel_type = StringField(choices=[(x, x) for x in TUNNEL_TYPES],
required=False)
tunnel_local_address = StringField(required=False)
tunnel_remote_address = StringField(required=False)
project = ForeignKeyField(Project)
def __unicode__(self):
return "%s %s" % (self.interface.managed_object.name, self.name)
def iter_changed_datastream(self):
if config.datastream.enable_managedobject:
yield "managedobject", self.managed_object.id
@property
def effective_vc_domain(self):
return self.interface.effective_vc_domain
def get_profile(self):
if self.profile:
return self.profile
else:
return self.interface.profile
class InterfaceClassificationMatch(EmbeddedDocument):
# Field name
field = StringField(choices=[
("name", "name"),
("description", "description"),
("ip", "ip"),
("tagged", "tagged vlan"),
("untagged", "untagged vlan")
])
# Operation
op = StringField(choices=[
("eq", "Equals"),
("regexp", "RegExp"),
("in", "in")
])
#
value = StringField()
# "ip in"
prefix_table = ForeignKeyField(PrefixTable, required=False)
# *vlan in
vc_filter = ForeignKeyField(VCFilter, required=False)
description = StringField(required=False)
def __unicode__(self):
if self.prefix_table:
v = self.prefix_table.name
elif self.vc_filter:
v = self.vc_filter.name
else:
v = self.value
return "%s %s %s" % (self.field, self.op, v)
def compile(self, f_name):
a = getattr(self, "compile_%s_%s" % (self.field, self.op), None)
if a:
return a(f_name)
else:
raise SyntaxError("%s %s is not implemented" % (
self.field, self.op))
# name
def compile_name_eq(self, f_name):
return "\n".join([
"def %s(iface):" % f_name,
" return iface.name.lower() == %s" % repr(self.value.lower())
])
def compile_name_regexp(self, f_name):
return "\n".join([
"rx_%s = re.compile(%s, re.IGNORECASE)" % (f_name, repr(self.value)),
"def %s(iface):" % f_name,
" return bool(rx_%s.search(iface.name))" % f_name
])
# description
def compile_description_eq(self, f_name):
return "\n".join([
"def %s(iface):" % f_name,
" return iface.description.lower() == %s" % repr(self.value.lower())
])
def compile_description_regexp(self, f_name):
return "\n".join([
"rx_%s = re.compile(%s, re.IGNORECASE)" % (f_name, repr(self.value)),
"def %s(iface):" % f_name,
" return iface.description and bool(rx_%s.search(iface.description))" % f_name
])
# IP
def compile_ip_eq(self, f_name):
v = IP.prefix(self.value)
r = [
"def %s(iface):" % f_name,
" a = [si.ipv%(afi)s_addresses for si in iface.subinterface_set.filter(enabled_afi='IPv%(afi)s')]" % {"afi": v.afi},
" a = sum(a, [])",
]
if "/" in self.value:
# Compare prefixes
r += [
" return any(x for x in a if x == %r)" % v.prefix
]
else:
# Compare addresses
v = v.prefix.split("/")[0]
r += [
" return any(x for x in a if x.split('/')[0] == %r)" % v
]
return "\n".join(r)
def compile_ip_in(self, f_name):
r = [
"pt_%s = PrefixTable.objects.get(id=%s)" % (f_name, self.prefix_table.id),
"def %s(iface):" % f_name,
" for si in iface.subinterface_set.filter(enabled_afi='IPv4'):",
" for a in si.ipv4_addresses:",
" if a in pt_%s:" % f_name,
" return True",
" for si in iface.subinterface_set.filter(enabled_afi='IPv6'):",
" for a in si.ipv6_addresses:",
" if a in pt_%s:" % f_name,
" return True",
" return False"
]
return "\n".join(r)
## Untagged
def compile_untagged_eq(self, f_name):
vlan = int(self.value)
if vlan < 1 or vlan > 4096:
raise SyntaxError("Invalid VLAN")
r = [
"def %s(iface):" % f_name,
" return bool(iface.subinterface_set.filter(enabled_afi='BRIDGE', untagged_vlan=%d).count())" % vlan
]
return "\n".join(r)
def compile_untagged_in(self, f_name):
r = [
"vcf_%s = VCFilter.objects.get(id=%s)" % (f_name, self.vc_filter.id),
"def %s(iface):" % f_name,
" for si in iface.subinterface_set.filter(enabled_afi='BRIDGE'):",
" if si.untagged_vlan and vcf_%s.check(si.untagged_vlan):" % f_name,
" return True",
" return False"
]
return "\n".join(r)
## Tagged
def compile_tagged_eq(self, f_name):
vlan = int(self.value)
if vlan < 1 or vlan > 4096:
raise SyntaxError("Invalid VLAN")
r = [
"def %s(iface):" % f_name,
" return bool(iface.subinterface_set.filter(enabled_afi='BRIDGE', tagged_vlans=%d).count())" % vlan
]
return "\n".join(r)
def compile_tagged_in(self, f_name):
r = [
"vcf_%s = VCFilter.objects.get(id=%s)" % (f_name, self.vc_filter.id),
"def %s(iface):" % f_name,
" for si in iface.subinterface_set.filter(enabled_afi='BRIDGE'):",
" if si.tagged_vlans:",
" if any(vlan for vlan in si.tagged_vlans if vcf_%s.check(vlan)):" % f_name,
" return True",
" return False"
]
return "\n".join(r)
class InterfaceClassificationRule(Document):
meta = {
"collection": "noc.inv.interfaceclassificationrules",
"allow_inheritance": False
}
name = StringField(required=False)
is_active = BooleanField(default=True)
description = StringField(required=False)
order = IntField()
selector = ForeignKeyField(ManagedObjectSelector, required=False)
match = ListField(
EmbeddedDocumentField(InterfaceClassificationMatch),
required=False)
profile = PlainReferenceField(InterfaceProfile,
default=InterfaceProfile.get_default_profile)
def __unicode__(self):
r = [unicode(x) for x in self.match]
return "%s -> %s" % (", ".join(r), self.profile.name)
@property
def match_expr(self):
"""
Stringified match expression
"""
if not len(self.match):
return u"any"
elif len(self.match) == 1:
return unicode(self.match[0])
else:
return u" AND ".join(u"(%s)" % unicode(m) for m in self.match)
@classmethod
def get_classificator_code(cls):
r = ["import re"]
mf = [
"gsc = {}",
"def classify(interface):",
" def in_selector(o, s):",
" if s in s_cache:",
" return s_cache[s]",
" if s in gsc:",
" selector = gsc[s]",
" else:",
" selector = ManagedObjectSelector.objects.get(id=s)",
" gsc[s] = selector",
" r = o in selector",
" s_cache[s] = r",
" return r",
" s_cache = {}",
" mo = interface.managed_object"
]
for rule in cls.objects.filter(is_active=True).order_by("order"):
rid = str(rule.id)
lmn = []
for i, m in enumerate(rule.match):
mn = "match_%s_%d" % (rid, i)
r += [m.compile(mn)]
lmn += ["%s(interface)" % mn]
if lmn:
mf += [
" if in_selector(mo, %d) and %s:" % (rule.selector.id, " and ".join(lmn)),
" return %r" % rule.profile.name
]
else:
mf += [" return %r" % rule.profile.name]
r += mf
return "\n".join(r)
@classmethod
def get_classificator(cls):
code = cls.get_classificator_code() + "\nhandlers[0] = classify\n"
# Hack to retrieve reference
handlers = {}
# Compile code
exec code in {
"re": re,
"PrefixTable": PrefixTable,
"VCFilter": VCFilter,
"ManagedObjectSelector": ManagedObjectSelector,
"handlers": handlers
}
return handlers[0]
class DiscoveryID(Document):
"""
Managed Object's discovery identity
"""
meta = {
"collection": "noc.inv.discovery_id",
"allow_inheritance": False,
"indexes": ["object", "hostname", "udld_id"]
}
object = ForeignKeyField(ManagedObject)
chassis_mac = ListField(EmbeddedDocumentField(MACRange))
hostname = StringField()
router_id = StringField()
udld_id = StringField() # UDLD local identifier
def __unicode__(self):
return self.object.name
@classmethod
def submit(cls, object, chassis_mac=None,
hostname=None, router_id=None):
if chassis_mac:
chassis_mac = [
MACRange(
first_mac=r["first_chassis_mac"],
last_mac=r["last_chassis_mac"]
) for r in chassis_mac
]
o = cls.objects.filter(object=object.id).first()
if o:
o.chassis_mac = chassis_mac
o.hostname = hostname
o.router_id = router_id
o.save()
else:
cls(object=object, chassis_mac=chassis_mac,
hostname=hostname, router_id=router_id).save()
@classmethod
def find_object(cls, mac=None):
"""
Find managed object
:param cls:
:return: Managed object instance or None
"""
c = cls._get_collection()
# Find by mac
if mac:
r = c.find_one({
"chassis_mac": {
"$elemMatch": {
"first_mac": {
"$lte": mac
},
"last_mac": {
"$gte": mac
}
}
}
})
if r:
return ManagedObject.objects.get(id=r["object"])
# Fallback to interface search
o = set()
for i in Interface.objects.filter(mac=mac):
o.add(i.managed_object)
if len(o) == 1:
return o.pop()
return None
@classmethod
def macs_for_object(cls, object):
"""
Get MAC addresses for object
:param cls:
:param object:
:return: list of (fist_mac, last_mac)
"""
try:
o = cls.objects.get(object=object.id)
except DoesNotExist:
return []
if not o or not o.chassis_mac:
return None
# Discovered chassis id range
c_macs = [(r.first_mac, r.last_mac) for r in o.chassis_mac]
# Other interface macs
i_macs = set()
for i in Interface.objects.filter(
managed_object=object.id, mac__exists=False):
if i.mac:
if not any(1 for f, t in c_macs if f <= i.mac <= t):
# Not in range
i_macs.add(i.mac)
return c_macs + [(m, m) for m in i_macs]
class MACDB(Document):
"""
Customer MAC address database
"""
meta = {
"collection": "noc.macs",
"allow_inheritance": False,
"indexes": ["mac", "interface"]
}
# Todo: Add Validation
mac = StringField()
vc_domain = ForeignKeyField(VCDomain, required=False)
vlan = IntField()
managed_object = ForeignKeyField(ManagedObject)
interface = PlainReferenceField(Interface)
last_changed = DateTimeField()
def __unicode__(self):
return self.mac
def save(self):
self.mac = MAC(self.mac)
if not self.last_changed:
self.last_changed = datetime.datetime.now()
super(MACDB, self).save()
@classmethod
def submit(cls, mac, vc_domain, vlan, interface, timestamp=None):
"""
Submit mac to database
Returns True if database been changed
:param cls:
:param mac:
:param interface:
:param timestamp:
:return:
"""
if not timestamp:
timestamp = datetime.datetime.now()
managed_object = interface.managed_object
mac = MAC(mac)
vcd = vc_domain.id if vc_domain else None
m = MACDB.objects.filter(mac=mac, vc_domain=vcd).first()
if m:
if (managed_object != m.managed_object or interface != m.interface
or vlan != m.vlan):
# Database change, write history
MACLog(timestamp=m.last_changed,
mac=mac,
vc_domain_name=vc_domain.name if vc_domain else None,
vlan=m.vlan,
managed_object_name=m.managed_object.name,
interface_name=m.interface.name).save()
m.vlan = vlan
m.managed_object = managed_object
m.interface = interface
m.last_changed = timestamp
m.save()
return True
else:
return False
else:
MACDB(mac=mac,
vc_domain=vc_domain,
vlan=vlan,
managed_object=managed_object,
interface=interface,
last_changed=timestamp).save()
return True
class MetricSettings(Document):
meta = {
"collection": "noc.pm.metricsettings",
"indexes": [("model_id", "object_id")]
}
# Reference to model or document, like sa.ManagedObject
model_id = StringField()
# Object id, converted to string
object_id = StringField()
# List of metric sets
metric_sets = ListField(EmbeddedDocumentField(MetricSettingsItem))
_model_cache = {} # model id -> model class
_document_cache = {} # model id -> document
def __unicode__(self):
return u"%s:%s" % (self.model_id, self.object_id)
def _init_document_cache(self):
for v in _document_registry.itervalues():
n = "%s.%s" % (v.__module__.split(".")[1], v.__name__)
self._document_cache[n] = v
from noc.pm.models.metricconfig import MetricConfig
self._document_cache["pm.MetricConfig"] = MetricConfig
def get_model(self):
m = self._model_cache.get(self.model_id)
if not m:
# Try django model
m = get_model(*self.model_id.split("."))
if not m:
if not self._document_cache:
self._init_document_cache()
# Try mongoengine model
m = self._document_cache[self.model_id]
self._model_cache[self.model_id] = m
return m
def get_object(self):
m = self.get_model()
try:
return m.objects.get(id=self.object_id)
except m.DoesNotExist:
return None
@classmethod
def get_model_id(cls, object):
if isinstance(object._meta, dict):
# Document
return u"%s.%s" % (object.__module__.split(".")[1],
object.__class__.__name__)
else:
# Model
return u"%s.%s" % (object._meta.app_label,
object._meta.object_name)
@classmethod
def get_settings(cls, object):
"""
Find MetricSettings instance
"""
return cls.objects.filter(model_id=cls.get_model_id(object),
object_id=str(object.pk)).first()
@classmethod
def get_effective_settings(cls, object, trace=False, recursive=False):
"""
Returns a list of effective settings for object
"""
def get_config(name):
if name in cvars:
v = cvars[name]
if isinstance(v, ValueError):
raise v
else:
try:
v = gc(name)
cvars[name] = v
except ValueError, why:
cvars[name] = ValueError(why)
raise cvars[name]
return v
def get_recursive():
r = []
handler = getattr(object, "iter_recursive_objects", None)
if recursive and handler:
for o in handler():
r += cls.get_effective_settings(o,
trace=trace,
recursive=True)
return r
s_seq = []
# Check profiles
model_id = cls.get_model_id(object)
p_field = getattr(object, "PROFILE_LINK", None)
if p_field:
p = getattr(object, p_field)
if p:
ps = cls.get_settings(p)
if ps:
s_seq += [ps]
# Check object's settings
s = cls.get_settings(object)
if s:
s_seq += [s]
if not s_seq:
return get_recursive()
mt = {} # metric type -> metric item
mti = {} # metric type -> interval
for s in s_seq:
for ms in s.metric_sets:
if not ms.is_active:
continue
for mi in ms.metric_set.get_effective_metrics():
mt[mi.metric_type] = mi
mti[mi.metric_type] = ms.metric_set.interval
r = []
cvars = {}
gc = getattr(object, "get_probe_config", None)
# Pass through router solution
for m, mi in mt.iteritems():
if not mi.is_active:
continue
try:
mo = get_config("managed_object")
except ValueError:
mo = None
es = EffectiveSettings(object=object,
model_id=model_id,
metric=None,
metric_type=m,
is_active=True,
probe=None,
managed_object=mo,
interval=mti[m],
thresholds=[
mi.low_error, mi.low_warn, mi.high_warn,
mi.high_error
])
_router(object, es)
if not es.is_active:
es.error("Deactivated by router")
if trace:
r += [es]
continue
if not es.metric:
es.error("No graphite metric found")
if trace:
r += [es]
continue
if not es.probe:
es.error("Not assigned to probe daemon")
if trace:
r += [es]
continue
if not es.probe.storage:
es.errors("No assigned storage")
if trace:
r += [es]
continue
# Get handler
for h in probe_registry.iter_handlers(m.name):
if trace:
es.trace("Checking %s" % h.handler_name)
config = {}
failed = False
# Check required parameters
if h.req:
if gc:
for name in h.req:
try:
config[name] = get_config(name)
except ValueError:
failed = True
if trace:
es.trace(
"Cannot get required variable '%s'" %
name)
break
else:
continue
if failed:
if trace:
es.trace("Giving up")
continue
# Get optional parameters
if gc:
for name in h.opt:
try:
config[name] = get_config(name)
except ValueError:
continue
# Handler found
if h.match(config):
es.handler = h.handler_name
es.config = config
es.convert = h.convert
es.scale = h.scale
if trace:
es.trace("Matched handler %s(%s)" %
(h.handler_name, config))
break
elif trace:
es.trace("Handler mismatch")
#
es.is_active = bool(es.handler)
if trace and not es.handler:
if not gc:
es.error("No get_probe_config method for %s" % model_id)
es.error("No handler found")
if es.is_active or trace:
r += [es]
# Collapse around handlers
rr = {}
for es in r:
probe_id = es.probe.id if es.probe else None
if es.handler:
key = (probe_id, es.handler, es.interval)
else:
key = (probe_id, es.metric, es.metric_type, es.interval)
if key in rr:
e = rr[key]
e.metrics += [
EffectiveSettingsMetric(metric=es.metric,
metric_type=es.metric_type,
thresholds=es.thresholds,
convert=es.convert,
scale=es.scale)
]
else:
es.metrics = [
EffectiveSettingsMetric(metric=es.metric,
metric_type=es.metric_type,
thresholds=es.thresholds,
convert=es.convert,
scale=es.scale)
]
es.metric = None
es.metric_type = None
es.thresholds = None
es.convert = None
es.scale = None
rr[key] = es
return rr.values() + get_recursive()
class Interface(Document):
"""
Interfaces
"""
meta = {
"collection": "noc.interfaces",
"allow_inheritance": False,
"indexes": [
("managed_object", "name"),
"mac",
("managed_object", "ifindex")
]
}
managed_object = ForeignKeyField(ManagedObject)
name = StringField() # Normalized via Profile.convert_interface_name
type = StringField(choices=[(x, x) for x in INTERFACE_TYPES])
description = StringField(required=False)
ifindex = IntField(required=False)
mac = StringField(required=False)
aggregated_interface = PlainReferenceField("self", required=False)
enabled_protocols = ListField(StringField(
choices=[(x, x) for x in INTERFACE_PROTOCOLS]
), default=[])
# @todo: admin status + oper status
profile = PlainReferenceField(InterfaceProfile,
default=InterfaceProfile.get_default_profile)
# profile locked on manual user change
profile_locked = BooleanField(required=False, default=False)
#
project = ForeignKeyField(Project)
state = ForeignKeyField(ResourceState)
vc_domain = ForeignKeyField(VCDomain)
# Coverage
coverage = PlainReferenceField(Coverage)
technologies = ListField(StringField())
PROFILE_LINK = "profile"
def __unicode__(self):
return u"%s: %s" % (self.managed_object.name, self.name)
def save(self, *args, **kwargs):
self.name = self.managed_object.profile.convert_interface_name(self.name)
if self.mac:
self.mac = MACAddressParameter().clean(self.mac)
super(Interface, self).save(*args, **kwargs)
def delete(self, *args, **kwargs):
# Remove all subinterfaces
for si in self.subinterface_set.all():
si.delete()
# Unlink
link = self.link
if link:
self.unlink()
# Flush MACDB
MACDB.objects.filter(interface=self.id).delete()
# Remove interface
super(Interface, self).delete(*args, **kwargs)
@property
def link(self):
"""
Return Link instance or None
:return:
"""
return Link.objects.filter(interfaces=self.id).first()
@property
def is_linked(self):
"""
Check interface is linked
:returns: True if interface is linked, False otherwise
"""
return bool(Link.objects.filter(interfaces=self.id).limit(1).count())
def unlink(self):
"""
Remove existing link.
Raise ValueError if interface is not linked
"""
link = self.link
if link is None:
raise ValueError("Interface is not linked")
if link.is_ptp:
link.delete()
else:
raise ValueError("Cannot unlink non p-t-p link")
def link_ptp(self, other, method=""):
"""
Create p-t-p link with other interface
Raise ValueError if either of interface already connected.
:type other: Interface
:returns: Link instance
"""
# Try to check existing LAG
el = Link.objects.filter(interfaces=self.id).first()
if el and other not in el.interfaces:
el = None
if (self.is_linked or other.is_linked) and not el:
raise ValueError("Already linked")
if self.id == other.id:
raise ValueError("Cannot link with self")
if self.type in ("physical", "management"):
if other.type in ("physical", "management"):
# Refine LAG
if el:
left_ifaces = [i for i in el.interfaces if i not in (self, other)]
if left_ifaces:
el.interfaces = left_ifaces
el.save()
else:
el.delete()
#
link = Link(interfaces=[self, other],
discovery_method=method)
link.save()
return link
else:
raise ValueError("Cannot connect %s interface to %s" % (
self.type, other.type))
elif self.type == "aggregated":
# LAG
if other.type == "aggregated":
# Check LAG size match
# Skip already linked members
l_members = [i for i in self.lag_members if not i.is_linked]
r_members = [i for i in other.lag_members if not i.is_linked]
if len(l_members) != len(r_members):
raise ValueError("LAG size mismatch")
# Create link
if l_members:
link = Link(interfaces=l_members + r_members,
discovery_method=method)
link.save()
return link
else:
return
else:
raise ValueError("Cannot connect %s interface to %s" % (
self.type, other.type))
raise ValueError("Cannot link")
@classmethod
def get_interface(cls, s):
"""
Parse <managed object>@<interface> string
and return interface instance or None
"""
if "@" not in s:
raise ValueError("Invalid interface: %s" % s)
o, i = s.rsplit("@", 1)
# Get managed object
try:
mo = ManagedObject.objects.get(name=o)
except ManagedObject.DoesNotExist:
raise ValueError("Invalid manged object: %s" % o)
# Normalize interface name
i = mo.profile.convert_interface_name(i)
# Look for interface
iface = Interface.objects.filter(managed_object=mo.id,
name=i).first()
return iface
@property
def subinterface_set(self):
return SubInterface.objects.filter(interface=self.id)
@property
def lag_members(self):
if self.type != "aggregated":
raise ValueError("Cannot net LAG members for not-aggregated interface")
return Interface.objects.filter(aggregated_interface=self.id)
@property
def effective_vc_domain(self):
if self.type in ("null", "tunnel", "other", "unknown"):
return None
if self.vc_domain:
return self.vc_domain
if self.managed_object.vc_domain:
return self.managed_object.vc_domain
return VCDomain.get_default()
def get_probe_config(self, config):
# Get via solutions
try:
return get_probe_config(self, config)
except ValueError:
pass
# Fallback
if config == "interface__name":
return self.name
elif config == "interface__ifindex":
if self.ifindex is None:
raise ValueError("No ifindex for %s" % self)
else:
return self.ifindex
try:
return self.managed_object.get_probe_config(config)
except ValueError:
pass
# Fallback to interface profile
return self.profile.get_probe_config(config)
class Link(Document):
"""
Network links.
Always contains a list of 2*N references.
2 - for fully resolved links
2*N for unresolved N-link portchannel
N, N > 2 - broadcast media
"""
meta = {
"collection": "noc.links",
"strict": False,
"auto_create_index": False,
"indexes": ["interfaces", "linked_objects", "linked_segments"]
}
# Optional link name
name = StringField()
# Optional description
description = StringField()
# Optional shape
shape = StringField()
# List of interfaces
interfaces = PlainReferenceListField("inv.Interface")
# Link type, detected automatically
type = StringField(
choices=[
# 2 managed objects, 2 linked interfaces
("p", "Point-to-Point"),
# 2 managed objects, even number of linked interfaces (>2)
("a", "Point-to-Point Aggregated"),
# >2 managed objects, one uplink
("m", "Point-to-Multipoint"),
# >2 managed objects, no dedicated uplink
("M", "Multipoint-to-Multipoint"),
# Unknown
("u", "Unknown")
],
default="u")
# List of linked objects
linked_objects = ListField(IntField())
# List of linked segments
linked_segments = ListField(ObjectIdField())
# Name of discovery method or "manual"
discovery_method = StringField()
# Timestamp of first discovery
first_discovered = DateTimeField(default=datetime.datetime.now)
# Timestamp of last confirmation
last_seen = DateTimeField()
# L2 path cost
l2_cost = IntField(default=1)
# L3 path cost
l3_cost = IntField(default=1)
def __unicode__(self):
if self.interfaces:
return u"(%s)" % ", ".join(unicode(i) for i in self.interfaces)
else:
return u"Stale link (%s)" % self.id
def iter_changed_datastream(self):
if config.datastream.enable_managedobject:
for mo_id in self.linked_objects:
yield "managedobject", mo_id
def clean(self):
self.linked_objects = sorted(
set(i.managed_object.id for i in self.interfaces))
self.linked_segments = sorted(
set(i.managed_object.segment.id for i in self.interfaces))
self.type = self.get_type()
def contains(self, iface):
"""
Check link contains interface
:return: boolean
"""
return iface in self.interfaces
@property
def is_ptp(self):
"""
Check link is point-to-point link
:return:
"""
return self.type == "p" or self.type == "a"
@property
def is_lag(self):
"""
Check link is unresolved LAG
:return:
"""
return self.type == "p" or self.type == "a"
@property
def is_broadcast(self):
"""
Check link is broadcast media
:return:
"""
return not self.is_ptp and not self.is_lag
@property
def is_loop(self):
"""
Check link is looping to same object
:return:
"""
return len(self.linked_objects) == 1
@property
def interface_ids(self):
"""
Returns list of interface ids, avoiding dereference
:return:
"""
def q(i):
if hasattr(i, "id"):
return i.id
return i
return [q(iface) for iface in self._data.get("interfaces", [])]
def other(self, interface):
"""
Return other interfaces of the link
:param interface:
:return:
"""
return [i for i in self.interfaces if i.id != interface.id]
def other_ptp(self, interface):
"""
Return other interface of ptp link
:param interface:
:return:
"""
return self.other(interface)[0]
def touch(self, method=None):
"""
Touch last_seen
"""
now = datetime.datetime.now()
op = {"last_seen": now}
self.last_seen = now
if method:
self.discovery_method = method
op["discovery_method"] = method
# Do not save to prevent rebuilding topology
self._get_collection().update({"_id": self.id}, {"$set": op})
# self.save()
@classmethod
def object_links(cls, object):
return Link.objects.filter(linked_objects=object.id)
@classmethod
def object_links_count(cls, object):
return Link.objects.filter(linked_objects=object.id).count()
def on_save(self):
if not hasattr(
self,
"_changed_fields") or "interfaces" in self._changed_fields:
self.update_topology()
def on_delete(self):
self.update_topology()
@property
def managed_objects(self):
"""
List of connected managed objects
"""
from noc.sa.models.managedobject import ManagedObject
return list(ManagedObject.objects.filter(id__in=self.linked_objects))
@property
def segments(self):
"""
List of segments connected by link
:return:
"""
from noc.inv.models.networksegment import NetworkSegment
return list(NetworkSegment.objects.filter(id__in=self.linked_segments))
def update_topology(self):
for mo in self.managed_objects:
mo.update_topology()
def get_type(self):
"""
Detect link type
:return: Link type as value for .type
"""
n_objects = len(self.linked_objects)
n_interfaces = len(self.interfaces)
if n_objects == 2 and n_interfaces == 2:
return "p" # Point-to-point
if n_objects == 2 and n_interfaces > 2 and n_interfaces % 2 == 0:
d = defaultdict(int) # object -> count
for i in self.interfaces:
d[i.managed_object.id] += 1
k = d.keys()
if d[k[0]] == d[k[1]]:
return "a" # Point-to-Point aggregated
if n_objects > 2:
if self.type == "m":
return "m"
else:
return "M"
return "u"
class FTSQueue(Document):
meta = {"collection": "noc.fts_queue", "allow_inheritance": False}
object = StringField(unique=True) # <module>.<Model>:<id>
op = StringField(choices=[("U", "Update"), ("D", "Delete")])
models = {} # FTS models
def unicode(self):
return "%s:%s" % (self.object, self.op)
@classmethod
def schedule_update(cls, o):
oid = cls.get_id(o)
logger.debug("Scheduling FTS update for %s (%s)", oid, o)
cls._get_collection().update({"object": oid},
{"$set": {
"object": oid,
"op": "U"
}},
upsert=True)
cls.schedule_job()
@classmethod
def schedule_delete(cls, o):
oid = cls.get_id(o)
logger.debug("Scheduling FTS delete for %s (%s)", oid, o)
cls._get_collection().update({"object": oid},
{"$set": {
"object": oid,
"op": "D"
}},
upsert=True)
cls.schedule_job()
@classmethod
def schedule_job(cls):
# sliding_job("main.jobs", "main.update_index",
# delta=5, cutoff_delta=15)
pass
@classmethod
def on_update_model(cls, sender, instance, **kwargs):
cls.schedule_update(instance)
@classmethod
def on_delete_model(cls, sender, instance, **kwargs):
cls.schedule_delete(instance)
@classmethod
def get_id(cls, o):
return "%s:%s" % (o._meta, o.id)
@classmethod
def on_new_model(cls, sender, **kwargs):
if hasattr(sender, "get_index"):
logger.debug("Adding FTS index for %s", sender._meta)
cls.models[str(sender._meta)] = sender
django_signals.post_save.connect(cls.on_update_model,
sender=sender)
django_signals.post_delete.connect(cls.on_delete_model,
sender=sender)
@classmethod
def install(cls):
"""
Install signal handlers
"""
django_signals.class_prepared.connect(cls.on_new_model)
@classmethod
def get_object(cls, id):
"""
Get object by FTS id
"""
m, i = id.split(":")
if not m in cls.models:
return None
model = cls.models[m]
try:
return model.objects.get(id=int(i))
except model.DoesNotExist:
return None
class Link(Document):
"""
Network links.
Always contains a list of 2*N references.
2 - for fully resolved links
2*N for unresolved N-link portchannel
N, N > 2 - broadcast media
"""
meta = {
"collection": "noc.links",
"allow_inheritance": False,
"indexes": ["interfaces"]
}
interfaces = PlainReferenceListField(Interface)
discovery_method = StringField()
def __unicode__(self):
return u"(%s)" % ", ".join([unicode(i) for i in self.interfaces])
def contains(self, iface):
"""
Check link contains interface
:return: boolean
"""
return iface in self.interfaces
@property
def is_ptp(self):
"""
Check link is point-to-point link
:return:
"""
return len(self.interfaces) == 2
@property
def is_lag(self):
"""
Check link is unresolved LAG
:return:
"""
if self.is_ptp:
return True
d = defaultdict(int) # object -> count
for i in self.interfaces:
d[i.managed_object.id] += 1
if len(d) != 2:
return False
k = d.keys()
return d[k[0]] == d[k[1]]
@property
def is_broadcast(self):
"""
Check link is broadcast media
:return:
"""
return not self.is_ptp and not self.is_lag
@property
def is_loop(self):
"""
Check link is looping to same object
:return:
"""
if not self.is_ptp:
return False
i1, i2 = self.interfaces
return i1.managed_object == i2.managed_object
def other(self, interface):
"""
Return other interfaces of the link
:param interface:
:return:
"""
return [i for i in self.interfaces if i.id != interface.id]
def other_ptp(self, interface):
"""
Return other interface of ptp link
:param interface:
:return:
"""
return self.other(interface)[0]
@classmethod
def object_links(cls, object):
ifaces = Interface.objects.filter(
managed_object=object.id).values_list("id")
return cls.objects.filter(interfaces__in=ifaces)
@classmethod
def object_links_count(cls, object):
ifaces = Interface.objects.filter(
managed_object=object.id).values_list("id")
return cls.objects.filter(interfaces__in=ifaces).count()
@classmethod
def _update_pop_links(cls, sender, document, target=None, **kwargs):
for i in document.interfaces:
for o in Object.get_managed(i.managed_object):
pop = o.get_pop()
if pop:
refresh_schedule("main.jobs",
"inv.update_pop_links",
key=pop.id,
delta=5)
class MACVendor(Document):
"""
IEEE OUI database
"""
meta = {
"collection": "noc.macvendors",
"strict": False,
"auto_create_index": False
}
# 3 octets, hexadecimal, upper
oui = StringField(primary_key=True)
vendor = StringField()
DOWNLOAD_URL = "http://standards-oui.ieee.org/oui.txt"
@classmethod
def get_vendor(cls, mac):
"""
Returns vendor for MAC or None
"""
oui = mac.replace(":", "").upper()[:6]
d = MACVendor._get_collection().find_one({"_id": oui}, {
"_id": 0,
"vendor": 1
})
if d:
return d.get("vendor")
else:
return None
@classmethod
def update(cls):
import requests
# Get new values
new = {}
logger.info("Fetching new items from %s", cls.DOWNLOAD_URL)
r = requests.get(cls.DOWNLOAD_URL)
assert r.status_code == 200
for l in r.text.splitlines():
if "(hex)" in l:
oui, vendor = l.split("(hex)")
oui = oui.strip().replace("-", "").upper()
vendor = vendor.strip()
new[oui] = vendor
# Get old values
old = dict((d["_id"], d["vendor"])
for d in MACVendor._get_collection().find())
# Compare
collection = MACVendor._get_collection()
bulk = []
for oui, vendor in new.iteritems():
if oui in old:
if vendor != old[oui]:
logger.info("[%s] %s -> %s", oui, old[oui], vendor)
bulk += [
UpdateOne({"_id": oui}, {"$set": {
"vendor": vendor
}})
]
else:
logger.info("[%s] Add %s", oui, vendor)
bulk += [InsertOne({"_id": oui, "vendor": vendor})]
for oui in set(old) - set(new):
logger.info("[%s] Delete")
bulk += [DeleteOne({"_id": oui})]
if bulk:
logger.info("Commiting changes to database")
try:
r = collection.bulk_write(bulk, ordered=False)
logger.info("Database has been synced")
if r.acknowledged:
logger.info("Inserted: %d, Modify: %d, Deleted: %d",
r.inserted_count + r.upserted_count,
r.modified_count, r.deleted_count)
except BulkWriteError as e:
logger.error("Bulk write error: '%s'", e.details)
logger.error("Stopping check")