def _get_all_addresses(cls, mo_id):
from noc.inv.models.subinterface import SubInterface
r = []
for d in SubInterface._get_collection().find(
{
"managed_object": int(mo_id),
"ipv4_addresses": {
"$exists": True
}
}, {"ipv4_addresses"}):
for a in d.get("ipv4_addresses", []):
r += [str(a).split("/")[0]]
return r
def find_object(cls, mac=None, ipv4_address=None):
"""
Find managed object
:param mac:
:param ipv4_address:
:param cls:
:return: Managed object instance or None
"""
def has_ip(ip, addresses):
x = ip + "/"
for a in addresses:
if a.startswith(x):
return True
return False
# Find by mac
if mac:
metrics["discoveryid_mac_requests"] += 1
r = cls.get_by_mac(mac)
if r:
return ManagedObject.get_by_id(r["object"])
if ipv4_address:
metrics["discoveryid_ip_requests"] += 1
# Try router_id
d = DiscoveryID.objects.filter(router_id=ipv4_address).first()
if d:
metrics["discoveryid_ip_routerid"] += 1
return d.object
# Fallback to interface addresses
o = set(
d["managed_object"]
for d in SubInterface._get_collection().with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED).find(
{
"ipv4_addresses": {
"$gt": ipv4_address + "/",
"$lt": ipv4_address + "/99"
}
},
{
"_id": 0,
"managed_object": 1,
"ipv4_addresses": 1
},
) if has_ip(ipv4_address, d["ipv4_addresses"]))
if len(o) == 1:
metrics["discoveryid_ip_interface"] += 1
return ManagedObject.get_by_id(list(o)[0])
metrics["discoveryid_ip_failed"] += 1
return None
def get_constraint(self):
# type: () -> Optional[BaseConstraint]
"""
Get optional path constraint
:return:
"""
constraint = UpwardsConstraint()
for doc in SubInterface._get_collection().find(
{"interface": self.object.id}, {"_id": 0, "enabled_afi": 1, "untagged_vlan": 1}
):
if "BRIDGE" in doc["enabled_afi"] and doc.get("untagged_vlan"):
constraint &= VLANConstraint(vlan=doc["untagged_vlan"], strict=False)
break
return constraint
def is_valid_link(self, link):
# type: (Link) -> bool
bridged_mo = set() # type: Set[int]
tagged_mo = set() # type: Set[int]
untagged_mo = set() # type: Set[int]
l3_mo = set()
for doc in SubInterface._get_collection().find(
{"interface": {
"$in": link.interface_ids
}},
{
"_id": 0,
"managed_object": 1,
"enabled_afi": 1,
"untagged_vlan": 1,
"tagged_vlans": 1,
"vlan_ids": 1,
},
):
if "BRIDGE" in doc["enabled_afi"]:
bridged_mo.add(doc["managed_object"])
if doc.get("untagged_vlan") == self.vlan:
untagged_mo.add(doc["managed_object"])
if self.vlan in doc.get("tagged_vlans", []):
tagged_mo.add(doc["managed_object"])
if self.vlan in doc.get("vlan_ids", []):
l3_mo.add(doc["managed_object"])
if self.strict:
# Both ends must satisfy
if len(bridged_mo) > 1 and (
(self.allow_tagged and len(tagged_mo) > 1) or
(self.allow_untagged and len(untagged_mo) > 1)):
# Bridge-to-Bridge
return True
if (len(bridged_mo) == 1
and ((self.allow_tagged and tagged_mo) or
(self.allow_untagged and untagged_mo))
and len(l3_mo) == 1):
# L3 to bridge
return True
else:
if len(bridged_mo) == 1 and (
(self.allow_tagged and len(tagged_mo) > 1) or
(self.allow_untagged and len(untagged_mo) > 1)):
# Bridge-to-Bridge
return True
if len(l3_mo) == 1:
# L3 to bridge
return True
return False
def get_interfaces(self, afi, rd, exclude=None):
"""
Returns a list of SI
"""
def check_ipv4(a):
if (a.startswith("127.") or a.startswith("169.254")
or a.endswith("/32") or a.startswith("0.0.0.0")):
return False
else:
return True
def check_ipv6(a):
if a == "::1":
return False
else:
return True
exclude = exclude or []
si_fields = {
"_id": 0,
"name": 1,
"forwarding_instance": 1,
"managed_object": 1
}
if afi == self.IPv4:
check = check_ipv4
get_addresses = lambda x: x.get("ipv4_addresses", [])
AFI = "IPv4"
si_fields["ipv4_addresses"] = 1
elif afi == self.IPv6:
check = check_ipv6
get_addresses = lambda x: x.get("ipv6_addresses", [])
AFI = "IPv6"
si_fields["ipv6_addresses"] = 1
else:
raise NotImplementedError()
for si in SubInterface._get_collection().find({"enabled_afi": AFI},
si_fields):
if rd != self.get_rd(si["managed_object"],
si.get("forwarding_instance")):
continue
seen = set(exclude)
for a in [a for a in get_addresses(si) if check(a)]:
prefix = str(IP.prefix(a).first)
if prefix in seen:
continue
seen.add(prefix)
self.p_power[prefix] += 1
yield self.SI(si["managed_object"], si["name"],
si.get("forwarding_instance"), a, prefix)
def get_subinterfaces(self):
subs = defaultdict(list) # interface id -> [{"name":, "ifindex":}]
for si in (SubInterface._get_collection().with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED).find(
{"managed_object": self.object.id}, {
"name": 1,
"interface": 1,
"ifindex": 1
})):
subs[si["interface"]] += [{
"name": si["name"],
"ifindex": si.get("ifindex")
}]
return subs
def get_interface_untagged_constraint(self, iface, strict=False):
# type: (Interface) -> BaseConstraint
for doc in SubInterface._get_collection().find(
{"interface": iface.id},
{
"_id": 0,
"enabled_afi": 1,
"untagged_vlan": 1,
"tagged_vlans": 1
},
):
if "BRIDGE" not in doc["enabled_afi"]:
continue
if doc.get("untagged_vlan"):
return VLANConstraint(doc["untagged_vlan"], strict=strict)
raise ValueError("Cannot get untagged vlan from interface")
def extract(self):
mos_id = dict(ManagedObject.objects.filter().values_list(
"id", "bi_id"))
si = SubInterface._get_collection().with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED)
for sub in si.find({
"description": {
"$exists": True
}
}, {
"_id": 0,
"managed_object": 1,
"name": 1,
"description": 1
}).sort("managed_object"):
yield (mos_id[sub["managed_object"]], sub["name"],
sub["description"])
def _apply_forwarding_instances(mo, r):
instances = list(
sorted(
ForwardingInstance._get_collection().find(
{"managed_object": mo.id}),
key=operator.itemgetter("name"),
))
if not instances:
return
si_map = defaultdict(list) # type: DefaultDict[ObjectId, List[str]]
for doc in SubInterface._get_collection().find(
{"managed_object": mo.id}, {
"_id": 0,
"name": 1,
"forwarding_instance": 1
}):
fi = doc.get("forwarding_instance")
if fi:
si_map[fi] += [doc["name"]]
result = []
for fi in instances:
item = {
"name": fi["name"],
"type": fi["type"],
"subinterfaces": si_map[fi["_id"]]
}
rd = fi.get("rd")
if rd:
item["rd"] = rd
vpn_id = fi.get("vpn_id")
if vpn_id:
item["vpn_id"] = vpn_id
rt_export = fi.get("rt_export")
if rt_export:
item["rt_export"] = rt_export
rt_import = fi.get("rt_import")
if rt_import:
item["rt_import"] = rt_import
result += [item]
r["forwarding_instances"] = result
def _apply_interfaces(mo: ManagedObject, r):
# id -> (object id, name)
ifcache = {}
# Get interfaces
interfaces = sorted(
Interface._get_collection().find({"managed_object": mo.id}),
key=lambda x: alnum_key(x["name"]),
)
# Populate cache
for i in interfaces:
ifcache[i["_id"]] = (i["managed_object"], i["name"])
# Get subs
subs = defaultdict(list)
for s in SubInterface._get_collection().find({"managed_object": mo.id}):
subs[s["interface"]] += [s]
# Get links
links = defaultdict(list)
for link in Link._get_collection().find({"linked_objects": mo.id}):
for li in link.get("interfaces", []):
links[li] += [link]
# Populate cache with linked interfaces
if links:
for i in Interface._get_collection().find(
{"_id": {"$in": list(links)}}, {"_id": 1, "managed_object": 1, "name": 1}
):
ifcache[i["_id"]] = (i["managed_object"], i["name"])
# Get services
svc_ids = [i["service"] for i in interfaces if i.get("service")]
if svc_ids:
services = {svc.id: svc for svc in Service.objects.filter(id__in=svc_ids)}
else:
services = {}
# Populate
r["interfaces"] = [
ManagedObjectDataStream._get_interface(
i, subs[i["_id"]], links[i["_id"]], ifcache, set(mo.data.uplinks), services
)
for i in interfaces
]
def _apply_interfaces(mo, r):
# id -> (object id, name)
ifcache = {}
# Get interfaces
interfaces = sorted(
Interface._get_collection().find({"managed_object": mo.id}),
key=lambda x: alnum_key(x["name"]),
)
# Populate cache
for i in interfaces:
ifcache[i["_id"]] = (i["managed_object"], i["name"])
# Get subs
subs = defaultdict(list)
for s in SubInterface._get_collection().find({"managed_object":
mo.id}):
subs[s["interface"]] += [s]
# Get links
links = defaultdict(list)
for l in Link._get_collection().find({"linked_objects": mo.id}):
for li in l.get("interfaces", []):
links[li] += [l]
# Populate cache with linked interfaces
if links:
for i in Interface._get_collection().find(
{"_id": {
"$in": list(links)
}}, {
"_id": 1,
"managed_object": 1,
"name": 1
}):
ifcache[i["_id"]] = (i["managed_object"], i["name"])
# Populate
r["interfaces"] = [
ManagedObjectDataStream._get_interface(i, subs[i["_id"]],
links[i["_id"]], ifcache)
for i in interfaces
]
def submit_subinterface(
self,
forwarding_instance,
interface,
name,
description=None,
mac=None,
vlan_ids=None,
enabled_afi=[],
ipv4_addresses=[],
ipv6_addresses=[],
iso_addresses=[],
vpi=None,
vci=None,
enabled_protocols=[],
untagged_vlan=None,
tagged_vlans=[],
ifindex=None,
):
mac = mac or interface.mac
si = self.get_subinterface(interface, name)
if si:
changes = self.update_if_changed(
si,
{
"forwarding_instance": forwarding_instance,
"description": description,
"mac": mac,
"vlan_ids": vlan_ids,
"enabled_afi": enabled_afi,
"ipv4_addresses": ipv4_addresses,
"ipv6_addresses": ipv6_addresses,
"iso_addresses": iso_addresses,
"vpi": vpi,
"vci": vci,
"enabled_protocols": enabled_protocols,
"untagged_vlan": untagged_vlan,
"tagged_vlans": tagged_vlans,
# ip_unnumbered_subinterface
"ifindex": ifindex,
},
ignore_empty=["ifindex"],
)
self.log_changes("Subinterface '%s' has been changed" % name,
changes)
else:
self.logger.info("Creating subinterface '%s'", name)
si = SubInterface(
forwarding_instance=forwarding_instance,
interface=interface.id,
managed_object=self.object.id,
name=name,
description=description,
mac=mac,
vlan_ids=vlan_ids,
enabled_afi=enabled_afi,
ipv4_addresses=ipv4_addresses,
ipv6_addresses=ipv6_addresses,
iso_addresses=iso_addresses,
vpi=None,
vci=None,
enabled_protocols=enabled_protocols,
untagged_vlan=untagged_vlan,
tagged_vlans=tagged_vlans,
ifindex=ifindex,
)
si.save()
if mac:
self.interface_macs.add(mac)
return si
def submit_subinterface(self,
forwarding_instance,
interface,
name,
description=None,
mac=None,
vlan_ids=None,
enabled_afi=[],
ipv4_addresses=[],
ipv6_addresses=[],
iso_addresses=[],
vpi=None,
vci=None,
enabled_protocols=[],
untagged_vlan=None,
tagged_vlans=[],
ifindex=None):
mac = mac or interface.mac
si = SubInterface.objects.filter(interface=interface.id,
name=name).first()
if si:
changes = self.update_if_changed(
si,
{
"forwarding_instance": forwarding_instance,
"description": description,
"mac": mac,
"vlan_ids": vlan_ids,
"enabled_afi": enabled_afi,
"ipv4_addresses": ipv4_addresses,
"ipv6_addresses": ipv6_addresses,
"iso_addresses": iso_addresses,
"vpi": vpi,
"vci": vci,
"enabled_protocols": enabled_protocols,
"untagged_vlan": untagged_vlan,
"tagged_vlans": tagged_vlans,
# ip_unnumbered_subinterface
"ifindex": ifindex
},
ignore_empty=["ifindex"])
self.log_changes("Subinterface '%s' has been changed" % name,
changes)
else:
self.info("Creating subinterface '%s'" % name)
si = SubInterface(forwarding_instance=forwarding_instance,
interface=interface.id,
managed_object=self.object.id,
name=name,
description=description,
mac=mac,
vlan_ids=vlan_ids,
enabled_afi=enabled_afi,
ipv4_addresses=ipv4_addresses,
ipv6_addresses=ipv6_addresses,
iso_addresses=iso_addresses,
vpi=None,
vci=None,
enabled_protocols=enabled_protocols,
untagged_vlan=untagged_vlan,
tagged_vlans=tagged_vlans,
ifindex=ifindex)
si.save()
# Submit found addresses and prefixes
if "IPv4" in enabled_afi or "IPv6" in enabled_afi:
# Get VRF
vrf = vrf_cache.get_or_create(
self.object,
forwarding_instance.name if forwarding_instance else "default",
forwarding_instance.rd if forwarding_instance else "0:0")
if vrf is None:
self.info("Skipping unknown VRF '%s'" % vrf["name"])
else:
# Submit ipv4 addresses and prefixes
if "IPv4" in enabled_afi:
for a in ipv4_addresses:
self.prefix_report.submit(vrf,
a,
interface=si.name,
description=si.description)
self.ip_report.submit(vrf,
a.split("/")[0],
interface=si.name,
mac=si.mac)
# Submit ipv6 addresses and prefixes
if "IPv6" in enabled_afi:
for a in ipv6_addresses:
self.prefix_report.submit(vrf,
a,
interface=si.name,
description=si.description)
self.ip_report.submit(vrf,
a.split("/")[0],
interface=si.name,
mac=si.mac)
# Process dual-stacking
if (len(ipv4_addresses) == len(ipv6_addresses)
and len(ipv4_addresses) > 0):
for ipv4, ipv6 in zip(ipv4_addresses, ipv6_addresses):
self.prefix_report.submit_dual_stack(vrf, ipv4, ipv6)
return si
