def make_keys( self, *key_fields, with_filter: Optional[Callable[[Dict], bool]] = None, with_translate=None, with_inventory=None, ): if not len(self.source_records): get_logger().info( f"Collection {self.name}:{self.source_class.__name__}: inventory empty." ) return # if key_fields is provided, then update the instance attribute since # the Caller could invoke make_keys again without specifying key_fields # again. self.key_fields = key_fields or self.key_fields with_filter = with_filter if with_filter else lambda x: True with_translate = with_translate or (lambda x: x) kf_getter = itemgetter(*self.key_fields) if not with_inventory: self.items.clear() for rec in with_inventory or self.source_records: try: # allow the itemize function to return None, indicating that # this specific item should be skipped. if (item := self.itemize(rec)) is None: continue if not with_filter(item): continue except Exception as exc: import traceback raise RuntimeError( f"Collection {self.name}: itimized failed.\n" f"Record: {rec}\n" f"Exception: {str(exc)}\n" f"Traceback: {traceback.format_exc(limit=2)}" )
async def diff_collections( origin_col: Collection, target_col: Collection, diff_filter_cls: Type[DiffCollectionsFilter], **options, ) -> DiffResults: # noqa col_name = origin_col.name target_name = target_col.source.name origin_name = origin_col.source.name log = get_logger() if not diff_filter_cls: diff_filter_cls = DiffCollectionsFilter log.info( f"using default filter: fiels={origin_col.FIELDS}, keys={origin_col.KEY_FIELDS}" ) diff_filter = diff_filter_cls(origin=origin_col, target=target_col) if diff_filter.key_fields: origin_col.key_fields = target_col.key_fields = diff_filter.key_fields if diff_filter.fields: origin_col.fields = target_col.fields = diff_filter.fields # ------------------------------------------------------------------------- # Obtain Origin Collections # ------------------------------------------------------------------------- log.info(f"Fetching {origin_name}/{col_name} collection ...") orig_ff = options["origin_filter"] or diff_filter.origin_fetch_filter() await origin_col.fetch(filters=orig_ff) log.info( f"Fetched {origin_name}/{col_name}, fetched {len(origin_col.source_records)} records." ) origin_col.make_keys(with_filter=diff_filter.origin_key_filter) # ------------------------------------------------------------------------- # Obtain Target Collections # ------------------------------------------------------------------------- target_ff = diff_filter.target_fetch_filter() log.info(f"Fetching {target_name}/{col_name} collection ...") await target_col.fetch(filters=target_ff) log.info( f"Fetched {target_name}/{col_name}, fetched {len(target_col.source_records)} records." ) target_col.make_keys(with_filter=diff_filter.target_key_filter) return diff(origin=origin_col, target=target_col, fields=diff_filter.fields)
def get_collection(source: Source, name: str) -> Collection: cfg = get_config() if name not in cfg.collections: msg = f"Collection '{name}' not found in nauti configuration file." get_logger().error(msg) raise RuntimeError(msg) if ( cls := next( ( cls for cls in Collection.__subclasses__() if cls.name == name and cls.source_class.name == source.name ), None, ) ) is None: raise RuntimeError(f"ERROR:NOT-FOUND: nauti collection: {source.name}/{name}")
async def add_items(self): # ------------------------------------------------------------------------- # Now create each of the device records. Once the device records are # created, then go back and add the primary interface and ipaddress values # using the other collections. # ------------------------------------------------------------------------- ipf_col = self.origin nb_col = self.target missing = self.diff_res.missing log = get_logger() def _report_device(update, _res: Response): key, item = update if _res.is_error: log.error( f"FAIL: create device {item['hostname']}: {_res.text}") return log.info( f"CREATE:OK: device {item['hostname']} ... creating primary IP ... " ) nb_col.source_records.append(_res.json()) await nb_col.add_items(items=missing, callback=_report_device) await self._ensure_primary_ipaddrs(missing=missing) # ------------------------------------------------------------------------- # for each of the missing device records perform a "change request" on the # 'ipaddr' field. so that the primary IP will be assigned. # ------------------------------------------------------------------------- ipaddr_changes = { key: { "ipaddr": ipf_col.items[key]["ipaddr"] } for key in missing.keys() } def _report_primary(item, _res): # noqa key, fields = item rec = nb_col.items[key] ident = f"device {rec['hostname']} assigned primary-ip4" if _res.is_error: log.error(f"CREATE:FAIL: {ident}: {_res.text}") return log.info(f"CREATE:OK: {ident}.") await nb_col.update_items(items=ipaddr_changes, callback=_report_primary)
async def add_items(self): log = get_logger() def _report(item, res: Response): _key, _fields = item ident = f"{_fields['hostname']}, {_fields['interface']} -> {_fields['portchan']}" if res.is_error: log.error(f"CREATE:FAIL: {ident}, {res.text}.") return log.info(f"CREATE:OK: {ident}.") await self.target.add_items(self.diff_res.missing, callback=_report)
async def fetch_target(self): log = get_logger() devices = {item["hostname"] for item in self.origin.items.values()} tasks = [self.target.fetch(device=device) for device in devices] ident = f"{self.target.source.name}/{self.name}" log.info(f"Fetching {ident} collection ...") # TODO: remove hardcode limit to something configuraable await iawait(tasks, limit=100) log.info(f"Fetched {ident}, fetched {len(self.target.source_records)} records.") self.target.make_keys(with_filter=self.target_key_filter)
async def add_items(self): nb_col = self.target log = get_logger() def _done(_item, _res: Response): _key, _fields = _item _res.raise_for_status() ident = f"ipaddr {_fields['hostname']}, {_fields['interface']}, {_fields['ipaddr']}" log.info(f"CREATE:OK: {ident}") log.info("CREATE:BEGIN: Netbox ipaddrs ...") await nb_col.add_items(self.diff_res.missing, callback=_done) log.info("CREATE:DONE: Netbox ipaddrs.")
async def update_items(self): nb_col = self.target changes = self.diff_res.changes log = get_logger() def _done(_item, res: Response): _key, _changes = _item _hostname, _ifname = _key res.raise_for_status() log.info(f"UPDATE:OK: ipaddr {_hostname}, {_ifname}") log.info("UPDATE:BEGIN: Netbox ipaddrs ...") await nb_col.update_items(changes, callback=_done) log.info("UPDATE:DONE: Netbox ipaddrs.")
async def fetch_origin(self): log = get_logger() ident = f"{self.origin.source.name}/{self.name}" log.info(f"Fetching {ident} collection ...") orig_ff = self.options.get( "origin_filter") or self.origin_fetch_filter() await self.origin.fetch(filters=orig_ff) log.info( f"Fetched {ident}, fetched {len(self.origin.source_records)} records." ) self.origin.make_keys(with_filter=self.origin_key_filter)
async def add_items(self): nb_col = self.diff_res.target missing = self.diff_res.missing log = get_logger() fields_fn = itemgetter("hostname", "interface") def _done(_item, _res: Response): _key, _fields = _item _res.raise_for_status() _hostname, _if_name = fields_fn(_fields) log.info(f"CREATE:OK: interface {_hostname}, {_if_name}") log.info("CREATE:BEGIN: Netbox interfaces ...") await nb_col.add_items(items=missing, callback=_done) log.info("CREATE:DONE: Netbox interfaces.")
async def update_items(self): nb_col = self.target log = get_logger() def _report(_item, res: Response): _key, _ch_fields = _item _fields = nb_col.items[_key] ident = f"{_fields['hostname']}, {_fields['interface']} -> {_ch_fields['portchan']}" if res.is_error: log.error(f"CHANGE:FAIL: {ident}, {res.text}") return log.info(f"CHANGE:OK: {ident}") await nb_col.update_items(self.diff_res.changes, callback=_report)
async def delete_items(self): nb_col = self.target changes = self.diff_res.extras log = get_logger() fields_fn = itemgetter("hostname", "ipaddr") def _done(_item, res: Response): _key, _fields = _item _hostname, _ipaddr = fields_fn(_fields) res.raise_for_status() log.info(f"DELETE:OK: ipaddr {_hostname}, {_ipaddr}") log.info("DELETE:BEGIN: Netbox ipaddrs ...") await nb_col.delete_items(changes, callback=_done) log.info("DELETE:DONE: Netbox ipaddrs.")
async def add_items(self): log = get_logger() nb_col = self.target missing = self.diff_res.missing def _report(item, res: Response): _key, _fields = item ident = f"site {_fields['name']}" if res.is_error: log.error(f"CREATE {ident}: FAIL: {res.text}") return log.info(f"CREATE {ident}: OK") log.info("CREATE:BEGIN: Netbox sites ...") await nb_col.add_items(missing, callback=_report) log.info("CREATE:DONE: Netbox sites ...")
async def delete_items(self): nb_col = self.diff_res.target changes = self.diff_res.extras fields_fn = itemgetter("hostname", "interface") log = get_logger() def _done(_item, _res: Response): _key, _ch_fields = _item _fields = nb_col.items[_key] _hostname, _ifname = fields_fn(_fields) _res.raise_for_status() log.info(f"DELETE:OK: interface {_hostname}, {_ifname}") log.info("DELETE:BEGIN: Netbox interfaces ...") await nb_col.delete_items(items=changes, callback=_done) log.info("DELETE:DONE: Netbox interfaces.")
async def delete_items(self): """ Extras exist when an interface in Netbox is associated to a LAG, but that interface is not associated to the LAG in IPF. In these cases we need to remove the relationship between the NB interface->LAG. """ nb_col = self.target log = get_logger() def _report(_item, res: Response): _key, _ch_fields = _item _fields = nb_col.items[_key] ident = f"{_fields['hostname']}, {_fields['interface']} -x {_fields['portchan']}" if res.is_error: log.error(f"REMOVE:FAIL: {ident}, {res.text}.") return log.info(f"REMOVE:OK: {ident}.") await nb_col.delete_items(self.diff_res.extras, callback=_report)
async def update_items(self): # TODO: need to test out update to device items # ipf_col = self.origin nb_col = self.target changes = self.diff_res.changes log = get_logger() def _report(change, res: Response): ch_key, ch_fields = change ch_rec = nb_col.items[ch_key] ident = f"device {ch_rec['hostname']}" if res.is_error: log.error(f"CHANGE:FAIL: {ident}, {res.text}") return log.info(f"CHANGE:OK: {ident}") actual_changes = dict() missing_pri_ip = dict() update_primary_ip = nb_col.config.options.get("update_primary_ip", False) for key, key_change in changes.items(): rec = nb_col.items[key] if (ipaddr := key_change.pop("ipaddr", None)) is not None: if any(( rec["ipaddr"] == "", (ipaddr and (update_primary_ip is True)), )): key_change["ipaddr"] = ipaddr missing_pri_ip[key] = key_change if len(key_change): actual_changes[key] = key_change
async def add_items(self, items, callback: Optional[CollectionCallback] = None): client: NetboxClient = self.source.client device_records = await client.fetch_devices( hostname_list=(rec["hostname"] for rec in items.values()), key="name" ) log = get_logger() def _create_task(key, fields): hostname, if_name = key if hostname not in device_records: log.error(f"device {hostname} missing.") return None # TODO: set the interface type correctly based on some kind of mapping definition. # for now, use this name-basis for loopback, vlan, port-channel. if_type = { "vl": "virtual", # vlan "vx": "virtual", # vxlan "lo": "virtual", # loopback "po": "lag", # port-channel }.get(if_name[0:2].lower(), "other") return client.post( url="/dcim/interfaces/", json=dict( device=device_records[hostname]["id"], name=if_name, description=fields["description"], type=if_type, ), ) await self.source.update(updates=items, callback=callback, creator=_create_task)
async def _ensure_primary_ipaddrs(self, missing: dict): log = get_logger() ipf_col = self.origin ipf_col_ipaddrs = get_collection(source=ipf_col.source, name="ipaddrs") ipf_col_ifaces = get_collection(source=ipf_col.source, name="interfaces") # ------------------------------------------------------------------------- # we need to fetch all of the IPF ipaddr records so that we can bind the # management IP address to the Netbox device record. We use the **IPF** # collection as the basis for the missing records so that the filter values # match. This is done to avoid any mapping changes that happended via the # collection intake process. This code is a bit of 'leaky-abstration', # so TODO: cleanup. # ------------------------------------------------------------------------- log.info("Fetching IP Fabric IP records ...") tasks = [ ipf_col_ipaddrs.fetch( filters= f"and(hostname = '{_item['hostname']}', ip = '{_item['loginIp']}')" ) for _item in [ipf_col.source_record_keys[key] for key in missing.keys()] ] await iawait(tasks, limit=50) ipf_col_ipaddrs.make_keys() # ------------------------------------------------------------------------- # now we need to gather the IPF interface records so we have any _fields that # need to be stored into Netbox (e.g. description) # ------------------------------------------------------------------------- log.info("Fetching IP Fabric interface records ...") tasks = [ ipf_col_ifaces.fetch( hostname=_item["hostname"], filters= f"and(hostname = '{_item['hostname']}', intName = '{_item['intName']}')", ) for _item in ipf_col_ipaddrs.source_record_keys.values() ] await iawait(tasks, limit=50) ipf_col_ifaces.make_keys() # ------------------------------------------------------------------------- # At this point we have the IPF collections for the needed 'interfaces' and # 'ipaddrs'. We need to ensure these same entities exist in the Netbox # collections. We will first attempt to find all the existing records in # Netbox using the `fetch_keys` method. # ------------------------------------------------------------------------- nb_col = self.target nb_col_ifaces = get_collection(source=nb_col.source, name="interfaces") nb_col_ipaddrs = get_collection(source=nb_col.source, name="ipaddrs") await nb_col_ifaces.fetch_items(items=ipf_col_ifaces.items) await nb_col_ipaddrs.fetch_items(items=ipf_col_ipaddrs.items) nb_col_ipaddrs.make_keys() nb_col_ifaces.make_keys() diff_ifaces = diff(origin=ipf_col_ifaces, target=nb_col_ifaces) diff_ipaddrs = diff(origin=ipf_col_ipaddrs, target=nb_col_ipaddrs) def _report_iface(item, _res: Response): _key, _fields = item hname, iname = _fields["hostname"], _fields["interface"] if _res.is_error: log.error( f"CREATE:FAIL: interface {hname}, {iname}: {_res.text}") return log.info(f"CREATE:OK: interface {hname}, {iname}.") nb_col_ifaces.source_records.append(_res.json()) if diff_ifaces.missing: await nb_col_ifaces.add_items(items=diff_ifaces.missing, callback=_report_iface) def _report_ipaddr(item, _res: Response): _key, _fields = item hname, iname, ipaddr = ( _fields["hostname"], _fields["interface"], _fields["ipaddr"], ) ident = f"ipaddr {hname}, {iname}, {ipaddr}" if _res.is_error: log.error(f"CREATE:FAIL: {ident}: {_res.text}") return nb_col_ipaddrs.source_records.append(_res.json()) log.info(f"CREATE:OK: {ident}.") if diff_ipaddrs.missing: await nb_col_ipaddrs.add_items(items=diff_ipaddrs.missing, callback=_report_ipaddr) nb_col.make_keys() nb_col_ifaces.make_keys() nb_col_ipaddrs.make_keys() # TODO: Note that I am passing the cached collections of interfaces and ipaddress # To the device collection to avoid duplicate lookups for record # indexes. Will give this approach some more thought. nb_col.cache["interfaces"] = nb_col_ifaces nb_col.cache["ipaddrs"] = nb_col_ipaddrs
async def update_items(self): get_logger().warning("Site changes not supported at this time.")
async def delete_items(self): get_logger().warning("Site removal not supported at this time.")
# ensure that there is a sync task registered for this # origin/target/collection, and if not exit with error. auditor = Auditor.get_registered( origin=origin, target=target, collection=collection, name=options["auditor"] ) auditor.options = options loop = asyncio.get_event_loop() diff_res = loop.run_until_complete(run_audit(auditor)) diff_report(diff_res, reports=options.get("diff_report")) if not (sync_opts := options["sync_action"]): return # Determine if there is a reconciler class registered for this combination. # if not, then report the error and exit. if (reconciler := Reconciler.get_registered(diff_res, name="default")) is None: get_logger().error("Missing registered reconcile task") return # Execute the sync reconcile process. reconciler.options = sync_opts loop.run_until_complete(run_reconcile(reconciler)) # all done.