def make_keys(
self,
*key_fields,
with_filter: Optional[Callable[[Dict], bool]] = None,
with_translate=None,
with_inventory=None,
):
if not len(self.source_records):
get_logger().info(
f"Collection {self.name}:{self.source_class.__name__}: inventory empty."
)
return
# if key_fields is provided, then update the instance attribute since
# the Caller could invoke make_keys again without specifying key_fields
# again.
self.key_fields = key_fields or self.key_fields
with_filter = with_filter if with_filter else lambda x: True
with_translate = with_translate or (lambda x: x)
kf_getter = itemgetter(*self.key_fields)
if not with_inventory:
self.items.clear()
for rec in with_inventory or self.source_records:
try:
# allow the itemize function to return None, indicating that
# this specific item should be skipped.
if (item := self.itemize(rec)) is None:
continue
if not with_filter(item):
continue
except Exception as exc:
import traceback
raise RuntimeError(
f"Collection {self.name}: itimized failed.\n"
f"Record: {rec}\n"
f"Exception: {str(exc)}\n"
f"Traceback: {traceback.format_exc(limit=2)}"
)
async def diff_collections(
origin_col: Collection,
target_col: Collection,
diff_filter_cls: Type[DiffCollectionsFilter],
**options,
) -> DiffResults: # noqa
col_name = origin_col.name
target_name = target_col.source.name
origin_name = origin_col.source.name
log = get_logger()
if not diff_filter_cls:
diff_filter_cls = DiffCollectionsFilter
log.info(
f"using default filter: fiels={origin_col.FIELDS}, keys={origin_col.KEY_FIELDS}"
)
diff_filter = diff_filter_cls(origin=origin_col, target=target_col)
if diff_filter.key_fields:
origin_col.key_fields = target_col.key_fields = diff_filter.key_fields
if diff_filter.fields:
origin_col.fields = target_col.fields = diff_filter.fields
# -------------------------------------------------------------------------
# Obtain Origin Collections
# -------------------------------------------------------------------------
log.info(f"Fetching {origin_name}/{col_name} collection ...")
orig_ff = options["origin_filter"] or diff_filter.origin_fetch_filter()
await origin_col.fetch(filters=orig_ff)
log.info(
f"Fetched {origin_name}/{col_name}, fetched {len(origin_col.source_records)} records."
)
origin_col.make_keys(with_filter=diff_filter.origin_key_filter)
# -------------------------------------------------------------------------
# Obtain Target Collections
# -------------------------------------------------------------------------
target_ff = diff_filter.target_fetch_filter()
log.info(f"Fetching {target_name}/{col_name} collection ...")
await target_col.fetch(filters=target_ff)
log.info(
f"Fetched {target_name}/{col_name}, fetched {len(target_col.source_records)} records."
)
target_col.make_keys(with_filter=diff_filter.target_key_filter)
return diff(origin=origin_col,
target=target_col,
fields=diff_filter.fields)
def get_collection(source: Source, name: str) -> Collection:
cfg = get_config()
if name not in cfg.collections:
msg = f"Collection '{name}' not found in nauti configuration file."
get_logger().error(msg)
raise RuntimeError(msg)
if (
cls := next(
(
cls
for cls in Collection.__subclasses__()
if cls.name == name and cls.source_class.name == source.name
),
None,
)
) is None:
raise RuntimeError(f"ERROR:NOT-FOUND: nauti collection: {source.name}/{name}")
async def add_items(self):
# -------------------------------------------------------------------------
# Now create each of the device records. Once the device records are
# created, then go back and add the primary interface and ipaddress values
# using the other collections.
# -------------------------------------------------------------------------
ipf_col = self.origin
nb_col = self.target
missing = self.diff_res.missing
log = get_logger()
def _report_device(update, _res: Response):
key, item = update
if _res.is_error:
log.error(
f"FAIL: create device {item['hostname']}: {_res.text}")
return
log.info(
f"CREATE:OK: device {item['hostname']} ... creating primary IP ... "
)
nb_col.source_records.append(_res.json())
await nb_col.add_items(items=missing, callback=_report_device)
await self._ensure_primary_ipaddrs(missing=missing)
# -------------------------------------------------------------------------
# for each of the missing device records perform a "change request" on the
# 'ipaddr' field. so that the primary IP will be assigned.
# -------------------------------------------------------------------------
ipaddr_changes = {
key: {
"ipaddr": ipf_col.items[key]["ipaddr"]
}
for key in missing.keys()
}
def _report_primary(item, _res): # noqa
key, fields = item
rec = nb_col.items[key]
ident = f"device {rec['hostname']} assigned primary-ip4"
if _res.is_error:
log.error(f"CREATE:FAIL: {ident}: {_res.text}")
return
log.info(f"CREATE:OK: {ident}.")
await nb_col.update_items(items=ipaddr_changes,
callback=_report_primary)
async def add_items(self):
log = get_logger()
def _report(item, res: Response):
_key, _fields = item
ident = f"{_fields['hostname']}, {_fields['interface']} -> {_fields['portchan']}"
if res.is_error:
log.error(f"CREATE:FAIL: {ident}, {res.text}.")
return
log.info(f"CREATE:OK: {ident}.")
await self.target.add_items(self.diff_res.missing, callback=_report)
async def fetch_target(self):
log = get_logger()
devices = {item["hostname"] for item in self.origin.items.values()}
tasks = [self.target.fetch(device=device) for device in devices]
ident = f"{self.target.source.name}/{self.name}"
log.info(f"Fetching {ident} collection ...")
# TODO: remove hardcode limit to something configuraable
await iawait(tasks, limit=100)
log.info(f"Fetched {ident}, fetched {len(self.target.source_records)} records.")
self.target.make_keys(with_filter=self.target_key_filter)
async def add_items(self):
nb_col = self.target
log = get_logger()
def _done(_item, _res: Response):
_key, _fields = _item
_res.raise_for_status()
ident = f"ipaddr {_fields['hostname']}, {_fields['interface']}, {_fields['ipaddr']}"
log.info(f"CREATE:OK: {ident}")
log.info("CREATE:BEGIN: Netbox ipaddrs ...")
await nb_col.add_items(self.diff_res.missing, callback=_done)
log.info("CREATE:DONE: Netbox ipaddrs.")
async def update_items(self):
nb_col = self.target
changes = self.diff_res.changes
log = get_logger()
def _done(_item, res: Response):
_key, _changes = _item
_hostname, _ifname = _key
res.raise_for_status()
log.info(f"UPDATE:OK: ipaddr {_hostname}, {_ifname}")
log.info("UPDATE:BEGIN: Netbox ipaddrs ...")
await nb_col.update_items(changes, callback=_done)
log.info("UPDATE:DONE: Netbox ipaddrs.")
async def fetch_origin(self):
log = get_logger()
ident = f"{self.origin.source.name}/{self.name}"
log.info(f"Fetching {ident} collection ...")
orig_ff = self.options.get(
"origin_filter") or self.origin_fetch_filter()
await self.origin.fetch(filters=orig_ff)
log.info(
f"Fetched {ident}, fetched {len(self.origin.source_records)} records."
)
self.origin.make_keys(with_filter=self.origin_key_filter)
async def add_items(self):
nb_col = self.diff_res.target
missing = self.diff_res.missing
log = get_logger()
fields_fn = itemgetter("hostname", "interface")
def _done(_item, _res: Response):
_key, _fields = _item
_res.raise_for_status()
_hostname, _if_name = fields_fn(_fields)
log.info(f"CREATE:OK: interface {_hostname}, {_if_name}")
log.info("CREATE:BEGIN: Netbox interfaces ...")
await nb_col.add_items(items=missing, callback=_done)
log.info("CREATE:DONE: Netbox interfaces.")
async def update_items(self):
nb_col = self.target
log = get_logger()
def _report(_item, res: Response):
_key, _ch_fields = _item
_fields = nb_col.items[_key]
ident = f"{_fields['hostname']}, {_fields['interface']} -> {_ch_fields['portchan']}"
if res.is_error:
log.error(f"CHANGE:FAIL: {ident}, {res.text}")
return
log.info(f"CHANGE:OK: {ident}")
await nb_col.update_items(self.diff_res.changes, callback=_report)
async def delete_items(self):
nb_col = self.target
changes = self.diff_res.extras
log = get_logger()
fields_fn = itemgetter("hostname", "ipaddr")
def _done(_item, res: Response):
_key, _fields = _item
_hostname, _ipaddr = fields_fn(_fields)
res.raise_for_status()
log.info(f"DELETE:OK: ipaddr {_hostname}, {_ipaddr}")
log.info("DELETE:BEGIN: Netbox ipaddrs ...")
await nb_col.delete_items(changes, callback=_done)
log.info("DELETE:DONE: Netbox ipaddrs.")
async def add_items(self):
log = get_logger()
nb_col = self.target
missing = self.diff_res.missing
def _report(item, res: Response):
_key, _fields = item
ident = f"site {_fields['name']}"
if res.is_error:
log.error(f"CREATE {ident}: FAIL: {res.text}")
return
log.info(f"CREATE {ident}: OK")
log.info("CREATE:BEGIN: Netbox sites ...")
await nb_col.add_items(missing, callback=_report)
log.info("CREATE:DONE: Netbox sites ...")
async def delete_items(self):
nb_col = self.diff_res.target
changes = self.diff_res.extras
fields_fn = itemgetter("hostname", "interface")
log = get_logger()
def _done(_item, _res: Response):
_key, _ch_fields = _item
_fields = nb_col.items[_key]
_hostname, _ifname = fields_fn(_fields)
_res.raise_for_status()
log.info(f"DELETE:OK: interface {_hostname}, {_ifname}")
log.info("DELETE:BEGIN: Netbox interfaces ...")
await nb_col.delete_items(items=changes, callback=_done)
log.info("DELETE:DONE: Netbox interfaces.")
async def delete_items(self):
"""
Extras exist when an interface in Netbox is associated to a LAG, but that
interface is not associated to the LAG in IPF. In these cases we need
to remove the relationship between the NB interface->LAG.
"""
nb_col = self.target
log = get_logger()
def _report(_item, res: Response):
_key, _ch_fields = _item
_fields = nb_col.items[_key]
ident = f"{_fields['hostname']}, {_fields['interface']} -x {_fields['portchan']}"
if res.is_error:
log.error(f"REMOVE:FAIL: {ident}, {res.text}.")
return
log.info(f"REMOVE:OK: {ident}.")
await nb_col.delete_items(self.diff_res.extras, callback=_report)
async def update_items(self):
# TODO: need to test out update to device items
# ipf_col = self.origin
nb_col = self.target
changes = self.diff_res.changes
log = get_logger()
def _report(change, res: Response):
ch_key, ch_fields = change
ch_rec = nb_col.items[ch_key]
ident = f"device {ch_rec['hostname']}"
if res.is_error:
log.error(f"CHANGE:FAIL: {ident}, {res.text}")
return
log.info(f"CHANGE:OK: {ident}")
actual_changes = dict()
missing_pri_ip = dict()
update_primary_ip = nb_col.config.options.get("update_primary_ip",
False)
for key, key_change in changes.items():
rec = nb_col.items[key]
if (ipaddr := key_change.pop("ipaddr", None)) is not None:
if any((
rec["ipaddr"] == "",
(ipaddr and (update_primary_ip is True)),
)):
key_change["ipaddr"] = ipaddr
missing_pri_ip[key] = key_change
if len(key_change):
actual_changes[key] = key_change
async def add_items(self, items, callback: Optional[CollectionCallback] = None):
client: NetboxClient = self.source.client
device_records = await client.fetch_devices(
hostname_list=(rec["hostname"] for rec in items.values()), key="name"
)
log = get_logger()
def _create_task(key, fields):
hostname, if_name = key
if hostname not in device_records:
log.error(f"device {hostname} missing.")
return None
# TODO: set the interface type correctly based on some kind of mapping definition.
# for now, use this name-basis for loopback, vlan, port-channel.
if_type = {
"vl": "virtual", # vlan
"vx": "virtual", # vxlan
"lo": "virtual", # loopback
"po": "lag", # port-channel
}.get(if_name[0:2].lower(), "other")
return client.post(
url="/dcim/interfaces/",
json=dict(
device=device_records[hostname]["id"],
name=if_name,
description=fields["description"],
type=if_type,
),
)
await self.source.update(updates=items, callback=callback, creator=_create_task)
async def _ensure_primary_ipaddrs(self, missing: dict):
log = get_logger()
ipf_col = self.origin
ipf_col_ipaddrs = get_collection(source=ipf_col.source, name="ipaddrs")
ipf_col_ifaces = get_collection(source=ipf_col.source,
name="interfaces")
# -------------------------------------------------------------------------
# we need to fetch all of the IPF ipaddr records so that we can bind the
# management IP address to the Netbox device record. We use the **IPF**
# collection as the basis for the missing records so that the filter values
# match. This is done to avoid any mapping changes that happended via the
# collection intake process. This code is a bit of 'leaky-abstration',
# so TODO: cleanup.
# -------------------------------------------------------------------------
log.info("Fetching IP Fabric IP records ...")
tasks = [
ipf_col_ipaddrs.fetch(
filters=
f"and(hostname = '{_item['hostname']}', ip = '{_item['loginIp']}')"
) for _item in
[ipf_col.source_record_keys[key] for key in missing.keys()]
]
await iawait(tasks, limit=50)
ipf_col_ipaddrs.make_keys()
# -------------------------------------------------------------------------
# now we need to gather the IPF interface records so we have any _fields that
# need to be stored into Netbox (e.g. description)
# -------------------------------------------------------------------------
log.info("Fetching IP Fabric interface records ...")
tasks = [
ipf_col_ifaces.fetch(
hostname=_item["hostname"],
filters=
f"and(hostname = '{_item['hostname']}', intName = '{_item['intName']}')",
) for _item in ipf_col_ipaddrs.source_record_keys.values()
]
await iawait(tasks, limit=50)
ipf_col_ifaces.make_keys()
# -------------------------------------------------------------------------
# At this point we have the IPF collections for the needed 'interfaces' and
# 'ipaddrs'. We need to ensure these same entities exist in the Netbox
# collections. We will first attempt to find all the existing records in
# Netbox using the `fetch_keys` method.
# -------------------------------------------------------------------------
nb_col = self.target
nb_col_ifaces = get_collection(source=nb_col.source, name="interfaces")
nb_col_ipaddrs = get_collection(source=nb_col.source, name="ipaddrs")
await nb_col_ifaces.fetch_items(items=ipf_col_ifaces.items)
await nb_col_ipaddrs.fetch_items(items=ipf_col_ipaddrs.items)
nb_col_ipaddrs.make_keys()
nb_col_ifaces.make_keys()
diff_ifaces = diff(origin=ipf_col_ifaces, target=nb_col_ifaces)
diff_ipaddrs = diff(origin=ipf_col_ipaddrs, target=nb_col_ipaddrs)
def _report_iface(item, _res: Response):
_key, _fields = item
hname, iname = _fields["hostname"], _fields["interface"]
if _res.is_error:
log.error(
f"CREATE:FAIL: interface {hname}, {iname}: {_res.text}")
return
log.info(f"CREATE:OK: interface {hname}, {iname}.")
nb_col_ifaces.source_records.append(_res.json())
if diff_ifaces.missing:
await nb_col_ifaces.add_items(items=diff_ifaces.missing,
callback=_report_iface)
def _report_ipaddr(item, _res: Response):
_key, _fields = item
hname, iname, ipaddr = (
_fields["hostname"],
_fields["interface"],
_fields["ipaddr"],
)
ident = f"ipaddr {hname}, {iname}, {ipaddr}"
if _res.is_error:
log.error(f"CREATE:FAIL: {ident}: {_res.text}")
return
nb_col_ipaddrs.source_records.append(_res.json())
log.info(f"CREATE:OK: {ident}.")
if diff_ipaddrs.missing:
await nb_col_ipaddrs.add_items(items=diff_ipaddrs.missing,
callback=_report_ipaddr)
nb_col.make_keys()
nb_col_ifaces.make_keys()
nb_col_ipaddrs.make_keys()
# TODO: Note that I am passing the cached collections of interfaces and ipaddress
# To the device collection to avoid duplicate lookups for record
# indexes. Will give this approach some more thought.
nb_col.cache["interfaces"] = nb_col_ifaces
nb_col.cache["ipaddrs"] = nb_col_ipaddrs
async def update_items(self):
get_logger().warning("Site changes not supported at this time.")
async def delete_items(self):
get_logger().warning("Site removal not supported at this time.")
# ensure that there is a sync task registered for this
# origin/target/collection, and if not exit with error.
auditor = Auditor.get_registered(
origin=origin, target=target, collection=collection, name=options["auditor"]
)
auditor.options = options
loop = asyncio.get_event_loop()
diff_res = loop.run_until_complete(run_audit(auditor))
diff_report(diff_res, reports=options.get("diff_report"))
if not (sync_opts := options["sync_action"]):
return
# Determine if there is a reconciler class registered for this combination.
# if not, then report the error and exit.
if (reconciler := Reconciler.get_registered(diff_res, name="default")) is None:
get_logger().error("Missing registered reconcile task")
return
# Execute the sync reconcile process.
reconciler.options = sync_opts
loop.run_until_complete(run_reconcile(reconciler))
# all done.
