class SubInterface(Document):
meta = {
"collection": "noc.subinterfaces",
"allow_inheritance": False,
"indexes": [
("managed_object", "ifindex"),
("managed_object", "vlan_ids"),
"interface", "managed_object",
"untagged_vlan", "tagged_vlans",
"enabled_afi"
]
}
interface = PlainReferenceField(Interface)
managed_object = ForeignKeyField(ManagedObject)
forwarding_instance = PlainReferenceField(
ForwardingInstance, required=False)
name = StringField()
description = StringField(required=False)
profile = PlainReferenceField(InterfaceProfile,
default=InterfaceProfile.get_default_profile)
mtu = IntField(required=False)
mac = StringField(required=False)
vlan_ids = ListField(IntField(), default=[])
enabled_afi = ListField(StringField(
choices=[(x, x) for x in SUBINTERFACE_AFI]
), default=[])
ipv4_addresses = ListField(StringField(), default=[])
ipv6_addresses = ListField(StringField(), default=[])
iso_addresses = ListField(StringField(), default=[])
vpi = IntField(required=False)
vci = IntField(required=False)
enabled_protocols = ListField(StringField(
choices=[(x, x) for x in SUBINTERFACE_PROTOCOLS]
), default=[])
untagged_vlan = IntField(required=False)
tagged_vlans = ListField(IntField(), default=[])
# ip_unnumbered_subinterface
ifindex = IntField(required=False)
# Tunnel services
tunnel_type = StringField(
choices=[(x, x) for x in TUNNEL_TYPES], required=False)
tunnel_local_address = StringField(required=False)
tunnel_remote_address = StringField(required=False)
project = ForeignKeyField(Project)
def __unicode__(self):
return "%s %s" % (self.interface.managed_object.name, self.name)
@property
def effective_vc_domain(self):
return self.interface.effective_vc_domain
def get_profile(self):
if self.profile:
return self.profile
else:
return self.interface.profile
class AddressProfile(Document):
meta = {
"collection": "addressprofiles",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
description = StringField()
# Address workflow
workflow = PlainReferenceField(Workflow)
style = ForeignKeyField(Style)
# Template.subject to render Address.name
name_template = ForeignKeyField(Template)
# Template.subject to render Address.fqdn
fqdn_template = ForeignKeyField(Template)
# Send seen event to prefix
seen_propagation_policy = StringField(choices=[("E", "Enable"),
("D", "Disable")],
default="D")
#
tags = ListField(StringField())
# Integration with external NRI and TT systems
# Reference to remote system object has been imported from
remote_system = PlainReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_name_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
return AddressProfile.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_name_cache"),
lock=lambda _: id_lock)
def get_by_name(cls, name):
return AddressProfile.objects.filter(name=name).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"),
lock=lambda _: id_lock)
def get_by_bi_id(cls, id):
return AddressProfile.objects.filter(bi_id=id).first()
class VPNProfile(Document):
meta = {
"collection": "vpnprofiles",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
description = StringField()
type = StringField(choices=[("vrf", "VRF"), ("vxlan", "VxLAN"),
("vpls", "VPLS"), ("vll", "VLL"),
("evpn", "EVPN"), ("ipsec", "IPSec"),
("gre", "GRE"), ("ipip", "IP-IP")],
default="vrf")
workflow = PlainReferenceField(Workflow)
# Template.subject to render VPN/VRF.name
name_template = ForeignKeyField(Template)
#
style = ForeignKeyField(Style)
# For vrf type -- default prefix profile
# Used to create AFI root prefixes
default_prefix_profile = PlainReferenceField("ip.PrefixProfile")
#
tags = ListField(StringField())
# Integration with external NRI and TT systems
# Reference to remote system object has been imported from
remote_system = PlainReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
return VPNProfile.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"),
lock=lambda _: id_lock)
def get_by_bi_id(cls, id):
return VPNProfile.objects.filter(bi_id=id).first()
def clean(self):
if self.type == "vrf" and not self.default_prefix_profile:
raise ValidationError(
"default_prefix_profile must be set for vrf type")
class MRTConfig(Document):
meta = {"collection": "noc.mrtconfig", "allow_inheritance": False}
name = StringField(unique=True)
is_active = BooleanField(default=True)
description = StringField(required=False)
permission_name = StringField(required=True)
selector = ForeignKeyField(ManagedObjectSelector, required=True)
reduce_pyrule = ForeignKeyField(PyRule, required=True)
map_script = StringField(required=True)
timeout = IntField(required=False)
def __unicode__(self):
return self.name
class CPEStatus(Document):
meta = {
"collection": "cpestatuses",
"strict": False,
"auto_create_index": False,
"indexes": [("managed_object", "interface")]
}
managed_object = ForeignKeyField(ManagedObject)
interface = StringField()
local_id = StringField()
global_id = StringField(unique=True)
name = StringField()
status = StringField()
type = StringField()
vendor = StringField()
model = StringField()
version = StringField()
serial = StringField()
ip = StringField()
mac = StringField()
modulation = StringField()
description = StringField()
location = StringField()
distance = IntField()
def __unicode__(self):
return self.global_id
class PrefixListCache(Document):
"""
Prepared prefix-list cache. Can hold IPv4/IPv6 prefixes at same time.
Prefixes are stored sorted
"""
meta = {
"collection": "noc.prefix_list_cache",
"strict": False,
"auto_create_index": False
}
peering_point = ForeignKeyField(PeeringPoint)
name = StringField()
prefixes = ListField(EmbeddedDocumentField(PrefixListCachePrefix))
changed = DateTimeField()
pushed = DateTimeField()
def __unicode__(self):
return u" %s/%s" % (self.peering_point.hostname, self.name)
def cmp_prefixes(self, prefixes):
"""
Compare cached prefixes with a list of (prefix, min, max)
"""
return [(c.prefix, c.min, c.max) for c in self.prefixes] == prefixes
class VPN(Document):
meta = {
"collection": "vpns",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
profile = PlainReferenceField(VPNProfile)
description = StringField()
state = PlainReferenceField(State)
# Link to parent overlay
parent = PlainReferenceField("self")
project = ForeignKeyField(Project)
route_target = ListField(EmbeddedDocumentField(RouteTargetItem))
tags = ListField(StringField())
# Integration with external NRI and TT systems
# Reference to remote system object has been imported from
remote_system = PlainReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
# @todo: last_seen
# @todo: expired
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
def get_by_id(cls, id):
return VPN.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock)
def get_by_bi_id(cls, id):
return VPN.objects.filter(bi_id=id).first()
def clean(self):
super(VPN, self).clean()
if self.id and "parent" in self._changed_fields and self.has_loop:
raise ValidationError("Creating VPN loop")
@property
def has_loop(self):
"""
Check if object creates loop
"""
if not self.id:
return False
p = self.parent
while p:
if p.id == self.id:
return True
p = p.parent
return False
class ForwardingInstance(Document):
"""
Non-default forwarding instances
"""
meta = {
"collection": "noc.forwardinginstances",
"allow_inheritance": False,
"indexes": ["managed_object"]
}
managed_object = ForeignKeyField(ManagedObject)
type = StringField(choices=[
(x, x) for x in ("ip", "bridge", "VRF", "VPLS", "VLL")
],
default="ip")
virtual_router = StringField(required=False)
name = StringField()
# VRF/VPLS
rd = StringField(required=False)
def __unicode__(self):
return u"%s: %s" % (self.managed_object.name,
self.name if self.name else "default")
def delete(self, *args, **kwargs):
# Delete subinterfaces
for si in self.subinterface_set.all():
si.delete()
# Delete forwarding instance
super(ForwardingInstance, self).delete(*args, **kwargs)
@property
def subinterface_set(self):
return SubInterface.objects.filter(forwarding_instance=self.id)
class ObjectCapabilities(Document):
meta = {"collection": "noc.sa.objectcapabilities", "indexes": ["object"]}
object = ForeignKeyField(ManagedObject)
caps = ListField(EmbeddedDocumentField(CapsItem))
def __unicode__(self):
return "%s caps" % self.object.name
class SLAProfile(Document):
"""
SLA profile and settings
"""
meta = {
"collection": "noc.sla_profiles",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
description = StringField()
#
style = ForeignKeyField(Style, required=False)
# Interface profile metrics
metrics = ListField(EmbeddedDocumentField(SLAProfileMetrics))
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
return SLAProfile.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_name(cls, name):
try:
return SLAProfile.objects.get(name=name)
except SLAProfile.DoesNotExist:
return None
class SubscriberProfile(Document):
meta = {
"collection": "noc.subscriberprofiles",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
description = StringField()
style = ForeignKeyField(Style, required=False)
workflow = PlainReferenceField(Workflow)
# FontAwesome glyph
glyph = StringField()
tags = ListField(StringField())
# Alarm weight
weight = IntField(default=0)
# Integration with external NRI and TT systems
# Reference to remote system object has been imported from
remote_system = PlainReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
return SubscriberProfile.objects.filter(id=id).first()
class VLANTranslation(EmbeddedDocument):
filter = ForeignKeyField(VCFilter)
rule = StringField(choices=[
# Rewrite tag to parent vlan's
("map", "map"),
# Append parent tag as S-VLAN
("push", "push")
], default="push")
parent_vlan = PlainReferenceField("vc.VLAN")
class PendingLinkCheck(Document):
"""
Customer MAC address changes
"""
meta = {
"collection": "noc.inv.pending_link_check",
"allow_inheritance": False,
"indexes": [("method", "local_object")]
}
method = StringField()
local_object = ForeignKeyField(ManagedObject)
local_interface = StringField() # optional
remote_object = ForeignKeyField(ManagedObject)
remote_interface = StringField()
expire = DateTimeField()
def __unicode__(self):
return u"%s:%s:%s:%s:%s" % (
self.method, self.local_object.name, self.local_interface,
self.remote_object.name, self.remote_interface)
@classmethod
def submit(cls, method, local_object, local_interface, remote_object,
remote_interface):
expire = datetime.datetime.now() + datetime.timedelta(days=2)
plc = PendingLinkCheck.objects.filter(
method=method,
local_object=local_object.id,
local_interface=local_interface,
remote_object=remote_object.id,
remote_interface=remote_interface).first()
if plc:
plc.expire = expire
else:
plc = cls(method=method,
local_object=local_object.id,
local_interface=local_interface,
remote_object=remote_object.id,
remote_interface=remote_interface,
expire=expire)
plc.save()
class ObjectFact(Document):
meta = {"collection": "noc.objectfacts", "indexes": ["object"]}
uuid = UUIDField(binary=True, primary_key=True)
object = ForeignKeyField(ManagedObject)
cls = StringField()
label = StringField()
attrs = DictField()
introduced = DateTimeField(default=datetime.datetime.now)
changed = DateTimeField(default=datetime.datetime.now)
def __unicode__(self):
return self.object.name
class AlarmSeverity(Document):
"""
Alarm severities
"""
meta = {
"collection": "noc.alarmseverities",
"allow_inheritance": False,
"indexes": ["severity"],
"json_collection": "fm.alarmseverities"
}
name = StringField(required=True, unique=True)
uuid = UUIDField(binary=True)
description = StringField(required=False)
severity = IntField(required=True)
style = ForeignKeyField(Style)
def __unicode__(self):
return self.name
@classmethod
def get_severity(cls, severity):
"""
Returns Alarm Severity instance corresponding to numeric value
"""
s = cls.objects.filter(
severity__lte=severity).order_by("-severity").first()
if not s:
s = cls.objects.order_by("severity").first()
return s
@classmethod
@cachetools.ttl_cache(maxsize=1000, ttl=600)
def get_severity_css_class_name(cls, severity):
return cls.get_severity(severity).style.css_class_name
def get_json_path(self):
return "%s.json" % quote_safe_path(self.name)
def to_json(self):
return to_json(
{
"name": self.name,
"$collection": self._meta["json_collection"],
"uuid": self.uuid,
"description": self.description,
"severity": self.severity,
"style__name": self.style.name
},
order=[
"name", "$collection", "uuid", "description", "severity",
"style"
])
class RouteTargetItem(EmbeddedDocument):
"""
Global (managed_object is None) or object-local VRF topology settings
"""
rd = StringField()
managed_object = ForeignKeyField(ManagedObject)
# forwarding_instance
op = StringField(choices=[
("both", "both"),
("import", "import"),
("export", "export")
])
target = StringField()
class FirmwarePolicy(Document):
meta = {
"collection": "noc.firmwarepolicy",
"strict": False,
"auto_create_index": False,
"indexes": ["platform", "firmware"]
}
# Platform (Matched with get_version)
platform = PlainReferenceField(Platform)
#
description = StringField()
#
object_profile = ForeignKeyField(ManagedObjectProfile)
#
firmware = PlainReferenceField(Firmware)
status = StringField(
choices=[(FS_RECOMMENDED, "Recommended"), (
FS_ACCEPTABLE,
"Acceptable"), (FS_NOT_RECOMMENDED,
"Not recommended"), (FS_DENIED, "Denied")])
#
management = ListField(EmbeddedDocumentField(ManagementPolicy))
@classmethod
def get_status(cls, platform, version):
if not platform or not version:
return None
fp = FirmwarePolicy.objects.filter(platform=platform.id,
firmware=version.id).first()
if fp:
return fp.status
else:
return None
@classmethod
def get_recommended_version(cls, platform):
if not platform:
return None
fp = FirmwarePolicy.objects.filter(platform=platform.id,
status=FS_RECOMMENDED).first()
if fp:
return fp.firmware.version
versions = []
for fp in FirmwarePolicy.objects.filter(platform=platform.id,
status=FS_ACCEPTABLE):
versions += [fp.firmware.version]
if versions:
# Get latest acceptable version
return sorted(versions, key=lambda x: split_alnum(x.name))[-1]
else:
return None
class Sync(Document):
meta = {
"collection": "noc.sync",
"allow_inheritance": False
}
name = StringField(unique=True)
is_active = BooleanField(default=True)
description = StringField()
user = ForeignKeyField(User)
n_instances = IntField(default=1)
def __unicode__(self):
return self.name
class FailedEvent(Document):
"""
Events that caused noc-classifier traceback
"""
meta = {
"collection": "noc.events.failed",
"strict": False,
"auto_create_index": False
}
status = "F"
# Fields
timestamp = DateTimeField(required=True)
managed_object = ForeignKeyField(ManagedObject, required=True)
source = StringField()
raw_vars = RawDictField(required=True)
# NOC version caused traceback
version = StringField(required=True)
traceback = StringField()
log = ListField(EmbeddedDocumentField(EventLog))
def __unicode__(self):
return u"%s" % self.id
def mark_as_new(self, message=None):
"""
Move to unclassified queue
"""
from noc.core.nsq.pub import nsq_pub
data = {"source": self.source}
data.update(self.raw_vars)
msg = {
"id": str(self.id),
"ts": time.mktime(self.timestamp.timetuple()),
"object": self.managed_object.id,
"data": data
}
nsq_pub("events.%s" % self.managed_object.pool.name, msg)
self.delete()
def log_message(self, message):
self.log += [
EventLog(timestamp=datetime.datetime.now(),
from_status=self.status,
to_status=self.status,
message=message)
]
self.save()
class SLAProbe(Document):
meta = {
"collection": "noc.sla_probes",
"strict": False,
"auto_create_index": False,
"indexes": [
"managed_object"
]
}
managed_object = ForeignKeyField(ManagedObject)
# Probe name (<managed object>, <group>, <name> triple must be unique
name = StringField()
# Probe profile
profile = PlainReferenceField(SLAProfile)
# Probe group
group = StringField()
# Optional description
description = StringField()
# Probe type
type = StringField(choices=[(x, x) for x in PROBE_TYPES])
# IP address or URL, depending on type
target = StringField()
# Hardware timestamps
hw_timestamp = BooleanField(default=False)
# Optional tags
tags = ListField(StringField())
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_target_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __unicode__(self):
return u"%s: %s" % (self.managed_object.name, self.name)
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
def get_by_id(cls, id):
return SLAProbe.objects.filter(id=id).first()
@cachetools.cachedmethod(operator.attrgetter("_target_cache"), lock=lambda _: id_lock)
def get_target(self):
mo = ManagedObject.objects.filter(address=self.target)[:1]
if mo:
return mo[0]
else:
return None
class Probe(Document):
"""
PM Probe daemon
"""
meta = {
"collection": "noc.pm.probe"
}
name = StringField(unique=True)
is_active = BooleanField(default=True)
description = StringField()
storage = ReferenceField(Storage)
user = ForeignKeyField(User)
n_instances = IntField(default=1)
def __unicode__(self):
return self.name
class VLANProfile(Document):
meta = {
"collection": "vlanprofiles",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
description = StringField()
# VLAN is management VLAN
enable_management = BooleanField(default=False)
# VLAN is multicast VLAN
enable_multicast = BooleanField(default=False)
# VLAN should be automatically provisioned
enable_provisioning = BooleanField(default=False)
# VLAN workflow
workflow = PlainReferenceField(Workflow)
style = ForeignKeyField(Style)
tags = ListField(StringField())
# Integration with external NRI and TT systems
# Reference to remote system object has been imported from
remote_system = PlainReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
def get_by_id(cls, id):
return VLANProfile.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock)
def get_by_bi_id(cls, id):
return VLANProfile.objects.filter(bi_id=id).first()
class Favorites(Document):
meta = {
"collection": "noc.favorites",
"strict": False,
"auto_create_index": False,
"indexes": ["user", ("user", "app")]
}
user = ForeignKeyField(User)
app = StringField()
favorite_app = BooleanField(default=False)
favorites = ListField()
def unicode(self):
return "%s:%s" % (self.user.username, self.app)
@classmethod
def add_item(cls, user, app_id, item):
fv = Favorites.objects.filter(
user=user.id, app=app_id).first()
if not fv:
fv = Favorites(user=user.id, app=app_id, favorites=[])
fi = list(fv.favorites) or []
if item not in fi:
logger.info("Setting favorite item %s@%s for user %s",
item, app_id, user.username)
fv.favorites = fi + [item]
fv.save()
@classmethod
def remove_item(cls, user, app_id, item):
fv = Favorites.objects.filter(
user=user.id, app=app_id).first()
fi = list(fv.favorites) or []
if fv and item and item in fi:
logger.info("Resetting favorite item %s@%s for user %s",
item, app_id, user.username)
fi.remove(item)
fv.favorites = fi
fv.save()
class InterfaceProfile(Document):
"""
Interface SLA profile and settings
"""
meta = {"collection": "noc.interface_profiles", "allow_inheritance": False}
name = StringField(unique=True)
description = StringField()
style = ForeignKeyField(Style, required=False)
# Interface-level events processing
link_events = StringField(required=True,
choices=[("I", "Ignore Events"),
("L",
"Log events, do not raise alarms"),
("A", "Raise alarms")],
default="A")
# Discovery settings
mac_discovery = BooleanField(default=False)
# check_link alarm job interval settings
# Either None or T0,I0,T1,I1,...,Tn-1,In-1,,In
# See MultiIntervalJob settings for details
check_link_interval = StringField(default=",60")
def __unicode__(self):
return self.name
@classmethod
def get_default_profile(cls):
try:
return cls._default_profile
except AttributeError:
cls._default_profile = cls.objects.filter(name="default").first()
return cls._default_profile
def get_probe_config(self, config):
try:
return get_probe_config(self, config)
except ValueError:
pass
raise ValueError("Invalid config '%s'" % config)
class PhoneNumberProfile(Document):
meta = {
"collection": "noc.phonenumberprofiles",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
description = StringField()
style = ForeignKeyField(Style)
workflow = PlainReferenceField(Workflow)
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
return PhoneNumberProfile.objects.filter(id=id).first()
class EscalationItem(EmbeddedDocument):
# Delay part
delay = IntField()
# Match part
administrative_domain = ForeignKeyField(AdministrativeDomain)
selector = ForeignKeyField(ManagedObjectSelector)
time_pattern = ForeignKeyField(TimePattern)
min_severity = IntField(default=0)
# Action part
notification_group = ForeignKeyField(NotificationGroup)
template = ForeignKeyField(Template)
clear_template = ForeignKeyField(Template)
create_tt = BooleanField(default=False)
close_tt = BooleanField(default=False)
wait_tt = BooleanField(default=False)
# Stop or continue to next rule
stop_processing = BooleanField(default=False)
class ASProfile(Document):
meta = {
"collection": "asprofiles",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
description = StringField()
style = ForeignKeyField(Style, required=False)
enable_discovery_prefix_whois_route = BooleanField(default=False)
prefix_profile_whois_route = PlainReferenceField("ip.PrefixProfile")
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
return ASProfile.objects.filter(id=id).first()
class Discovery(Document):
meta = {
"collection": "noc.schedules.inv.discovery",
"strict": False,
"auto_create_index": False
}
job_class = StringField(db_field='jcls')
schedule = DictField()
ts = DateTimeField(db_field='ts')
last = DateTimeField()
last_success = DateTimeField(db_field='st')
last_duration = FloatField(db_field='ldur')
last_status = StringField(db_field='ls')
status = StringField(db_field='s')
managed_object = ForeignKeyField(ManagedObject, db_field='key')
data = DictField()
traceback = DictField()
runs = IntField()
faults = IntField(db_field='f')
log = ListField()
def __unicode__(self):
return "%s: %s" % (self.managed_object, self.job_class)
class PrefixProfile(Document):
meta = {
"collection": "prefixprofiles",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
description = StringField()
# Enable nested Address discovery
# via ARP cache
enable_ip_discovery = BooleanField(default=False)
# Enable nested Addresses discovery
# via active PING probes
enable_ip_ping_discovery = BooleanField(default=False)
# Enable nested prefix prefix discovery
enable_prefix_discovery = BooleanField(default=False)
# Prefix workflow
workflow = PlainReferenceField(Workflow)
style = ForeignKeyField(Style)
# Template.subject to render Prefix.name
name_template = ForeignKeyField(Template)
# Discovery policies
prefix_discovery_policy = StringField(
choices=[
("E", "Enable"),
("D", "Disable")
], default="D"
)
address_discovery_policy = StringField(
choices=[
("E", "Enable"),
("D", "Disable")
], default="D"
)
# Send seen event to parent
seen_propagation_policy = StringField(
choices=[
("P", "Propagate"),
("E", "Enable"),
("D", "Disable")
], default="P"
)
#
tags = ListField(StringField())
# Integration with external NRI and TT systems
# Reference to remote system object has been imported from
remote_system = PlainReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
def get_by_id(cls, id):
return PrefixProfile.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock)
def get_by_bi_id(cls, id):
return PrefixProfile.objects.filter(bi_id=id).first()
class DiscoveryID(Document):
"""
Managed Object's discovery identity
"""
meta = {
"collection": "noc.inv.discovery_id",
"strict": False,
"auto_create_index": False,
"indexes": ["object", "hostname", "udld_id", "macs"]
}
object = ForeignKeyField(ManagedObject)
chassis_mac = ListField(EmbeddedDocumentField(MACRange))
hostname = StringField()
router_id = StringField()
udld_id = StringField() # UDLD local identifier
#
macs = ListField(LongField())
_mac_cache = cachetools.TTLCache(maxsize=10000, ttl=60)
_udld_cache = cachetools.TTLCache(maxsize=1000, ttl=60)
def __unicode__(self):
return self.object.name
@staticmethod
def _macs_as_ints(ranges=None, additional=None):
"""
Get all MAC addresses within ranges as integers
:param ranges: list of dicts {first_chassis_mac: ..., last_chassis_mac: ...}
:param additional: Optional list of additional macs
:return: List of integers
"""
ranges = ranges or []
additional = additional or []
# Apply ranges
macs = set()
for r in ranges:
if not r:
continue
first = MAC(r["first_chassis_mac"])
last = MAC(r["last_chassis_mac"])
macs.update(m for m in range(int(first), int(last) + 1))
# Append additional macs
macs.update(int(MAC(m)) for m in additional)
return sorted(macs)
@staticmethod
def _macs_to_ranges(macs):
"""
Convert list of macs (as integers) to MACRange
:param macs: List of integer
:return: List of MACRange
"""
ranges = []
for mi in macs:
if ranges and mi - ranges[-1][1] == 1:
# Extend last range
ranges[-1][1] = mi
else:
# New range
ranges += [[mi, mi]]
return [
MACRange(first_mac=str(MAC(r[0])), last_mac=str(MAC(r[1])))
for r in ranges
]
@classmethod
def submit(cls,
object,
chassis_mac=None,
hostname=None,
router_id=None,
additional_macs=None):
# Process ranges
macs = cls._macs_as_ints(chassis_mac, additional_macs)
ranges = cls._macs_to_ranges(macs)
# Update database
o = cls.objects.filter(object=object.id).first()
if o:
old_macs = set(m.first_mac for m in o.chassis_mac)
o.chassis_mac = ranges
o.hostname = hostname
o.router_id = router_id
old_macs -= set(m.first_mac for m in o.chassis_mac)
if old_macs:
cache.delete_many(["discoveryid-mac-%s" % m for m in old_macs])
# MAC index
o.macs = macs
o.save()
else:
cls(object=object,
chassis_mac=ranges,
hostname=hostname,
router_id=router_id,
macs=macs).save()
@classmethod
@cachedmethod(operator.attrgetter("_mac_cache"),
key="discoveryid-mac-%s",
lock=lambda _: mac_lock)
def get_by_mac(cls, mac):
return cls._get_collection().find_one({"macs": int(MAC(mac))}, {
"_id": 0,
"object": 1
})
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_udld_cache"),
lock=lambda _: mac_lock)
def get_by_udld_id(cls, device_id):
return cls._get_collection().find_one({"udld_id": device_id}, {
"_id": 0,
"object": 1
})
@classmethod
def find_object(cls, mac=None, ipv4_address=None):
"""
Find managed object
:param mac:
:param ipv4_address:
:param cls:
:return: Managed object instance or None
"""
def has_ip(ip, addresses):
x = ip + "/"
for a in addresses:
if a.startswith(x):
return True
return False
# Find by mac
if mac:
metrics["discoveryid_mac_requests"] += 1
r = cls.get_by_mac(mac)
if r:
return ManagedObject.get_by_id(r["object"])
if ipv4_address:
metrics["discoveryid_ip_requests"] += 1
# Try router_id
d = DiscoveryID.objects.filter(router_id=ipv4_address).first()
if d:
metrics["discoveryid_ip_routerid"] += 1
return d.object
# Fallback to interface addresses
o = set(
d["managed_object"]
for d in SubInterface._get_collection().with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED).find(
{
"ipv4_addresses": {
"$gt": ipv4_address + "/",
"$lt": ipv4_address + "/99"
}
}, {
"_id": 0,
"managed_object": 1,
"ipv4_addresses": 1
}) if has_ip(ipv4_address, d["ipv4_addresses"]))
if len(o) == 1:
metrics["discoveryid_ip_interface"] += 1
return ManagedObject.get_by_id(list(o)[0])
metrics["discoveryid_ip_failed"] += 1
return None
@classmethod
def macs_for_object(cls, object):
"""
Get MAC addresses for object
:param cls:
:param object:
:return: list of (fist_mac, last_mac)
"""
# Get discovered chassis id range
o = DiscoveryID.objects.filter(object=object.id).first()
if o and o.chassis_mac:
c_macs = [(r.first_mac, r.last_mac) for r in o.chassis_mac]
else:
c_macs = []
# Get interface macs
i_macs = set(i.mac for i in Interface.objects.filter(
managed_object=object.id, mac__exists=True).only("mac") if i.mac)
# Enrich discovered macs with additional interface's ones
c_macs += [(m, m) for m in i_macs
if not any(1 for f, t in c_macs if f <= m <= t)]
return c_macs
@classmethod
def macs_for_objects(cls, objects_ids):
"""
Get MAC addresses for object
:param cls:
:param objects_ids: Lis IDs of Managed Object Instance
:type: list
:return: Dictionary mac: objects
:rtype: dict
"""
if not objects_ids:
return None
if isinstance(objects_ids, list):
objects = objects_ids
else:
objects = list(objects_ids)
os = cls.objects.filter(object__in=objects)
if not os:
return None
# Discovered chassis id range
c_macs = {
int(did[0][0]): did[1]
for did in os.scalar("macs", "object") if did[0]
}
# c_macs = [r.macs for r in os]
# Other interface macs
i_macs = {
int(MAC(i[0])): i[1]
for i in Interface.objects.filter(managed_object__in=objects,
mac__exists=True).scalar(
"mac", "managed_object")
if i[0]
}
# Other subinterface macs (actual for DSLAM)
si_macs = {
int(MAC(i[0])): i[1]
for i in SubInterface.objects.filter(managed_object__in=objects,
mac__exists=True).scalar(
"mac", "managed_object")
if i[0]
}
c_macs.update(i_macs)
c_macs.update(si_macs)
return c_macs
def on_delete(self):
# Reset cache
macs = set(m.first_mac for m in self.chassis_mac)
if macs:
cache.delete_many(["discoveryid-mac-%s" % m for m in macs])
@classmethod
def clean_for_object(cls, mo):
if hasattr(mo, "id"):
mo = mo.id
for d in DiscoveryID.objects.filter(object=mo):
d.delete()
@classmethod
def find_objects(cls, macs):
"""
Find objects for list of macs
:param macs: List of MAC addresses
:return: dict of MAC -> ManagedObject for resolved MACs
"""
r = {}
if not macs:
return r
# Build list of macs to search
mlist = sorted(int(MAC(m)) for m in macs)
# Search for macs
obj_ranges = {} # (first, last) -> mo
for d in DiscoveryID._get_collection().find({"macs": {
"$in": mlist
}}, {
"_id": 0,
"object": 1,
"chassis_mac": 1
}):
mo = ManagedObject.get_by_id(d["object"])
if mo:
for dd in d.get("chassis_mac", []):
obj_ranges[int(MAC(dd["first_mac"])),
int(MAC(dd["last_mac"]))] = mo
n = 1
for s, e in obj_ranges:
n += 1
# Resolve ranges
start = 0
ll = len(mlist)
for s, e in sorted(obj_ranges):
mo = obj_ranges[s, e]
start = bisect.bisect_left(mlist, s, start, ll)
while start < ll and s <= mlist[start] <= e:
r[MAC(mlist[start])] = mo
start += 1
return r
@classmethod
def update_udld_id(cls, object, local_id):
"""
Update UDLD id if necessary
:param object: Object for set
:param local_id: Local UDLD id
:return:
"""
DiscoveryID._get_collection().update_one(
{"object": object.id}, {"$set": {
"udld_id": local_id
}},
upsert=True)