def _apply_interfaces(mo, r):
# id -> (object id, name)
ifcache = {}
# Get interfaces
interfaces = sorted(
Interface._get_collection().find({"managed_object": mo.id}),
key=lambda x: split_alnum(x["name"]),
)
# Populate cache
for i in interfaces:
ifcache[i["_id"]] = (i["managed_object"], i["name"])
# Get subs
subs = defaultdict(list)
for s in SubInterface._get_collection().find({"managed_object": mo.id}):
subs[s["interface"]] += [s]
# Get links
links = defaultdict(list)
for l in Link._get_collection().find({"linked_objects": mo.id}):
for li in l.get("interfaces", []):
links[li] += [l]
# Populate cache with linked interfaces
if links:
for i in Interface._get_collection().find(
{"_id": {"$in": list(links)}}, {"_id": 1, "managed_object": 1, "name": 1}
):
ifcache[i["_id"]] = (i["managed_object"], i["name"])
# Populate
r["interfaces"] = [
ManagedObjectDataStream._get_interface(i, subs[i["_id"]], links[i["_id"]], ifcache)
for i in interfaces
]
def _get_loopback_addresses(cls, mo_id):
from noc.inv.models.interface import Interface
from noc.inv.models.subinterface import SubInterface
# Get all loopbacks
if_ids = []
for d in Interface._get_collection().find(
{
"managed_object": int(mo_id),
"type": "loopback"
}, {"_id": 1}):
if_ids += [d["_id"]]
if not if_ids:
return []
# Get loopback's addresses
r = []
for d in SubInterface._get_collection().find(
{
"managed_object": int(mo_id),
"interface": {
"$in": if_ids
},
"ipv4_addresses": {
"$exists": True
},
},
{
"_id": 0,
"ipv4_addresses": 1
},
):
for a in d.get("ipv4_addresses", []):
r += [str(a).split("/")[0]]
return r
def _apply_forwarding_instances(mo: ManagedObject, r):
instances = list(
sorted(
ForwardingInstance._get_collection().find({"managed_object": mo.id}),
key=operator.itemgetter("name"),
)
)
if not instances:
return
si_map: DefaultDict[ObjectId, List[str]] = defaultdict(list)
for doc in SubInterface._get_collection().find(
{"managed_object": mo.id}, {"_id": 0, "name": 1, "forwarding_instance": 1}
):
fi = doc.get("forwarding_instance")
if fi:
si_map[fi] += [doc["name"]]
result = []
for fi in instances:
item = {"name": fi["name"], "type": fi["type"], "subinterfaces": si_map[fi["_id"]]}
rd = fi.get("rd")
if rd:
item["rd"] = rd
vpn_id = fi.get("vpn_id")
if vpn_id:
item["vpn_id"] = vpn_id
rt_export = fi.get("rt_export")
if rt_export:
item["rt_export"] = rt_export
rt_import = fi.get("rt_import")
if rt_import:
item["rt_import"] = rt_import
result += [item]
r["forwarding_instances"] = result
def get_subinterfaces(self):
subs = defaultdict(list) # interface id -> [{"name":, "ifindex":}]
for si in (
SubInterface._get_collection()
.with_options(read_preference=ReadPreference.SECONDARY_PREFERRED)
.find({"managed_object": self.object.id}, {"name": 1, "interface": 1, "ifindex": 1})
):
subs[si["interface"]] += [{"name": si["name"], "ifindex": si.get("ifindex")}]
return subs
def get_interfaces(self, afi, rd, exclude=None):
"""
Returns a list of SI
"""
def check_ipv4(a):
if (a.startswith("127.") or a.startswith("169.254")
or a.endswith("/32") or a.startswith("0.0.0.0")):
return False
else:
return True
def check_ipv6(a):
if a == "::1":
return False
else:
return True
exclude = exclude or []
si_fields = {
"_id": 0,
"name": 1,
"forwarding_instance": 1,
"managed_object": 1
}
if afi == self.IPv4:
check = check_ipv4
def get_addresses(x):
return x.get("ipv4_addresses", [])
AFI = "IPv4"
si_fields["ipv4_addresses"] = 1
elif afi == self.IPv6:
check = check_ipv6
def get_addresses(x):
return x.get("ipv6_addresses", [])
AFI = "IPv6"
si_fields["ipv6_addresses"] = 1
else:
raise NotImplementedError()
for si in SubInterface._get_collection().find({"enabled_afi": AFI},
si_fields):
if rd != self.get_rd(si["managed_object"],
si.get("forwarding_instance")):
continue
seen = set(exclude)
for a in [a for a in get_addresses(si) if check(a)]:
prefix = str(IP.prefix(a).first)
if prefix in seen:
continue
seen.add(prefix)
self.p_power[prefix] += 1
yield self.SI(si["managed_object"], si["name"],
si.get("forwarding_instance"), a, prefix)
def extract(self):
mos_id = dict(ManagedObject.objects.filter().values_list("id", "bi_id"))
si = SubInterface._get_collection().with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED
)
for sub in si.find(
{"description": {"$exists": True}},
{"_id": 0, "managed_object": 1, "name": 1, "description": 1},
).sort("managed_object"):
yield (mos_id[sub["managed_object"]], sub["name"], sub["description"])
def _get_all_addresses(cls, mo_id):
from noc.inv.models.subinterface import SubInterface
r = []
for d in SubInterface._get_collection().find(
{"managed_object": int(mo_id), "ipv4_addresses": {"$exists": True}}, {"ipv4_addresses"}
):
for a in d.get("ipv4_addresses", []):
r += [str(a).split("/")[0]]
return r
def get_interface_untagged_constraint(
self, iface: Interface, strict: bool = False
) -> BaseConstraint:
for doc in SubInterface._get_collection().find(
{"interface": iface.id},
{"_id": 0, "enabled_afi": 1, "untagged_vlan": 1, "tagged_vlans": 1},
):
if "BRIDGE" not in doc["enabled_afi"]:
continue
if doc.get("untagged_vlan"):
return VLANConstraint(doc["untagged_vlan"], strict=strict)
raise ValueError("Cannot get untagged vlan from interface")
def get_constraint(self):
# type: () -> Optional[BaseConstraint]
"""
Get optional path constraint
:return:
"""
constraint = UpwardsConstraint()
for doc in SubInterface._get_collection().find(
{"interface": self.object.id}, {"_id": 0, "enabled_afi": 1, "untagged_vlan": 1}
):
if "BRIDGE" in doc["enabled_afi"] and doc.get("untagged_vlan"):
constraint &= VLANConstraint(vlan=doc["untagged_vlan"], strict=False)
break
return constraint
def find_object(cls, mac=None, ipv4_address=None):
"""
Find managed object
:param mac:
:param ipv4_address:
:param cls:
:return: Managed object instance or None
"""
def has_ip(ip, addresses):
x = ip + "/"
for a in addresses:
if a.startswith(x):
return True
return False
# Find by mac
if mac:
metrics["discoveryid_mac_requests"] += 1
r = cls.get_by_mac(mac)
if r:
return ManagedObject.get_by_id(r["object"])
if ipv4_address:
metrics["discoveryid_ip_requests"] += 1
# Try router_id
d = DiscoveryID.objects.filter(router_id=ipv4_address).first()
if d:
metrics["discoveryid_ip_routerid"] += 1
return d.object
# Fallback to interface addresses
o = set(
d["managed_object"]
for d in SubInterface._get_collection().with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED).find(
{
"ipv4_addresses": {
"$gt": ipv4_address + "/",
"$lt": ipv4_address + "/99"
}
},
{
"_id": 0,
"managed_object": 1,
"ipv4_addresses": 1
},
) if has_ip(ipv4_address, d["ipv4_addresses"]))
if len(o) == 1:
metrics["discoveryid_ip_interface"] += 1
return ManagedObject.get_by_id(list(o)[0])
metrics["discoveryid_ip_failed"] += 1
return None
def is_valid_link(self, link):
# type: (Link) -> bool
bridged_mo = set() # type: Set[int]
tagged_mo = set() # type: Set[int]
untagged_mo = set() # type: Set[int]
l3_mo = set()
for doc in SubInterface._get_collection().find(
{"interface": {
"$in": link.interface_ids
}},
{
"_id": 0,
"managed_object": 1,
"enabled_afi": 1,
"untagged_vlan": 1,
"tagged_vlans": 1,
"vlan_ids": 1,
},
):
if "BRIDGE" in doc["enabled_afi"]:
bridged_mo.add(doc["managed_object"])
if doc.get("untagged_vlan") == self.vlan:
untagged_mo.add(doc["managed_object"])
if self.vlan in doc.get("tagged_vlans", []):
tagged_mo.add(doc["managed_object"])
if self.vlan in doc.get("vlan_ids", []):
l3_mo.add(doc["managed_object"])
if self.strict:
# Both ends must satisfy
if len(bridged_mo) > 1 and (
(self.allow_tagged and len(tagged_mo) > 1) or
(self.allow_untagged and len(untagged_mo) > 1)):
# Bridge-to-Bridge
return True
if (len(bridged_mo) == 1
and ((self.allow_tagged and tagged_mo) or
(self.allow_untagged and untagged_mo))
and len(l3_mo) == 1):
# L3 to bridge
return True
else:
if len(bridged_mo) == 1 and (
(self.allow_tagged and len(tagged_mo) > 1) or
(self.allow_untagged and len(untagged_mo) > 1)):
# Bridge-to-Bridge
return True
if len(l3_mo) == 1:
# L3 to bridge
return True
return False
def _apply_interfaces(mo: ManagedObject, r):
# id -> (object id, name)
ifcache = {}
# Get interfaces
interfaces = sorted(
Interface._get_collection().find({"managed_object": mo.id}),
key=lambda x: alnum_key(x["name"]),
)
# Populate cache
for i in interfaces:
ifcache[i["_id"]] = (i["managed_object"], i["name"])
# Get subs
subs = defaultdict(list)
for s in SubInterface._get_collection().find({"managed_object": mo.id}):
subs[s["interface"]] += [s]
# Get links
links = defaultdict(list)
for link in Link._get_collection().find({"linked_objects": mo.id}):
for li in link.get("interfaces", []):
links[li] += [link]
# Populate cache with linked interfaces
if links:
for i in Interface._get_collection().find(
{"_id": {"$in": list(links)}}, {"_id": 1, "managed_object": 1, "name": 1}
):
ifcache[i["_id"]] = (i["managed_object"], i["name"])
# Get services
svc_ids = [i["service"] for i in interfaces if i.get("service")]
if svc_ids:
services = {svc.id: svc for svc in Service.objects.filter(id__in=svc_ids)}
else:
services = {}
# Populate
r["interfaces"] = [
ManagedObjectDataStream._get_interface(
i, subs[i["_id"]], links[i["_id"]], ifcache, set(mo.data.uplinks), services
)
for i in interfaces
]
