def build_seen_objects(self, if_mac):
"""
Build seen_objects artefact
:param if_mac: interface -> [macs]
:return: interface -> [managed objects]
"""
# Resolve MACs
all_macs = reduce(lambda x, y: x | y, six.itervalues(if_mac))
mmap = DiscoveryID.find_objects(all_macs)
if not mmap:
self.logger.info(
"Cannot build seen_objects artefact: Cannot resolve any MACs")
return
# Bind resolved MACs to interfaces
seen_objects = defaultdict(set) # interface -> [ManagedObject]
for iface in if_mac:
rr = set(mmap[m] for m in if_mac[iface] if m in mmap)
if rr:
seen_objects[iface] = rr
# Update artifact
current = self.get_artefact("seen_objects") or {}
current.update(seen_objects)
self.set_artefact("seen_objects", current)
def handler(self):
self.logger.info("Checking %s topology", self.name)
# Get segment hierarchy
segments = set(self.object.get_nested_ids())
# Get managed objects and id <-> bi_id mappings
bi_map = {} # bi_id -> mo
for mo in ManagedObject.objects.filter(
segment__in=[str(x) for x in segments]
):
bi_map[str(mo.bi_id)] = mo
if not bi_map:
self.logger.info("Empty segment tree. Skipping")
return
# Fetch latest MAC tables snapshots from ClickHouse
# @todo: Apply vlan restrictions
t0 = datetime.datetime.now() - datetime.timedelta(seconds=self.MAC_WINDOW)
t0 = t0.replace(microsecond=0)
SQL = """SELECT managed_object, mac, argMax(ts, ts), argMax(interface, ts)
FROM mac
WHERE
date >= toDate('%s')
AND ts >= toDateTime('%s')
AND managed_object IN (%s)
GROUP BY ts, managed_object, mac
""" % (t0.date().isoformat(), t0.isoformat(sep=" "),
", ".join(bi_map))
ch = connection()
# Fill FIB
mtable = [] # mo_id, mac, iface, ts
last_ts = {} # mo -> ts
for mo_bi_id, mac, ts, iface in ch.execute(post=SQL):
mo = bi_map.get(mo_bi_id)
if mo:
mtable += [[mo, MAC(int(mac)), iface, ts]]
last_ts[mo] = max(ts, last_ts.get(mo, ts))
# Filter out aged MACs
mtable = [m for m in mtable if m[3] == last_ts[m[0]]]
# Resolve objects
macs = set(x[1] for x in mtable)
if not macs:
self.logger.info("No MAC addresses collected. Stopping")
return
object_macs = DiscoveryID.find_objects(macs)
if not object_macs:
self.logger.info("Cannot resolve any MAC addresses. Stopping")
return
# Build FIB
fib = {} # object -> interface -> {seen objects}
for mo, mac, iface, ts in mtable:
ro = object_macs.get(mac)
if not ro:
continue
if mo not in fib:
fib[mo] = {}
if iface in fib[mo]:
fib[mo][iface].add(ro)
else:
fib[mo][iface] = {ro}
# Find uplinks and coverage
coverage = {} # mo -> covered objects
uplinks = {} # mo -> uplink interface
up_fib = {} # mo -> {seen via uplinks}
for mo in fib:
coverage[mo] = {mo}
for iface in fib[mo]:
if self.is_uplink(mo, fib[mo][iface], segments):
uplinks[mo] = iface
up_fib[mo] = fib[mo][iface]
else:
coverage[mo] |= fib[mo][iface]
if mo not in uplinks:
self.logger.info(
"[%s] Cannot detect uplinks. Topology may be imprecise",
mo.name
)
# Dump FIB
if self.logger.isEnabledFor(logging.DEBUG):
for mo in fib:
self.logger.debug("%s:", mo.name)
if mo in uplinks:
self.logger.debug(" * %s: %s", uplinks[mo], ", ".join(x.name for x in up_fib[mo]))
else:
self.logger.debug(" Warning: No uplinks. Topology may be imprecise")
for iface in fib[mo]:
self.logger.debug(" %s: %s", iface, ", ".join(x.name for x in fib[mo][iface]))
# Build topology
for mo in fib:
for iface in fib[mo]:
if iface == uplinks.get(mo):
continue
for ro in fib[mo][iface]:
cvr = coverage.get(ro)
if not cvr:
cvr = {ro}
coverage[ro] = cvr
if not fib[mo][iface] - cvr:
# All objects from mo:iface are seen via ro
uplink = uplinks.get(ro)
if uplink:
self.confirm_link(mo, iface, ro, uplink)
break
else:
self.logger.info(
"[%s] No uplinks. Cannot link to %s:%s. Topology may be imprecise",
ro.name, mo.name, iface
)
def handle_vacuum_bulling(self, ids, *args, **kwargs):
connect()
for mo_id in ids:
mo = ManagedObject.get_by_id(mo_id)
if not mo:
self.print("@@@ %s is not found, skipping", mo_id)
continue
self.print("@@@ %s (%s, %s)", mo.name, mo.address, mo.id)
# Get interfaces suitable for bulling
bulling_ifaces: Set[Interface] = {
iface
for iface in Interface.objects.filter(managed_object=mo.id)
if not iface.profile.allow_vacuum_bulling
}
if not bulling_ifaces:
self.print("No interfaces suitable for vacuum bulling")
continue
# Get MAC addresses for bulling
t0 = datetime.datetime.now() - datetime.timedelta(
seconds=self.MAC_WINDOW)
t0 = t0.replace(microsecond=0)
sql = self.GET_MACS_SQL % (
mo.bi_id,
", ".join("'%s'" % iface.name.replace("'", "''")
for iface in bulling_ifaces),
t0.date().isoformat(),
t0.isoformat(sep=" "),
)
ch = connection()
last_ts: Optional[str] = None
all_macs: List[str] = []
mac_iface: Dict[str, str] = {}
for ts, iface, mac in ch.execute(post=sql):
if last_ts is None:
last_ts = ts
elif last_ts > ts:
continue
m = str(MAC(int(mac)))
all_macs += [m]
mac_iface[m] = iface
# Resolve MACs to known chassis-id
mac_map = DiscoveryID.find_objects(all_macs)
# Filter suitable rivals
seg_ifaces: DefaultDict[NetworkSegment,
Set[str]] = defaultdict(set)
iface_segs: DefaultDict[str,
Set[NetworkSegment]] = defaultdict(set)
for mac, r_mo in mac_map.items():
iface = mac_iface.get(mac)
if not iface:
continue
seg_ifaces[r_mo.segment].add(iface)
iface_segs[iface].add(r_mo.segment)
rej_ifaces: Set[str] = set()
for seg in seg_ifaces:
if len(seg_ifaces[seg]
) > 1 or seg.profile.is_persistent or seg == mo.segment:
# Seen on multiple interfaces or persistent segment or same segment
rej_ifaces |= set(seg_ifaces[seg])
continue
for iface in sorted(iface_segs, key=alnum_key):
if iface in rej_ifaces:
continue
for seg in iface_segs[iface]:
self.print(" '%s' challenging '%s' on %s" %
(mo.segment.name, seg.name, iface))
BioSegTrial.schedule_trial(seg, mo.segment)