async def ensure_devices(ipf, netbox, **params) -> IPFabricDeviceCollection:
"""
Ensure Netbox contains devices found IP Fabric in given Site
"""
print("\nEnsure Devices.")
print("Fetching from IP Fabric ... ", flush=True, end="")
ipf_col: IPFabricDeviceCollection = get_collection( # noqa
source=ipf, name="devices"
)
filters = params["filters"]
await ipf_col.fetch(filters=filters)
ipf_col.make_keys()
print(f"{len(ipf_col)} items.", flush=True)
if not len(ipf_col.source_records):
print(f"Done. No source_records matching filter:\n\t{filters}")
return ipf_col
print("Fetching from Netbox ... ", flush=True, end="")
netbox_col: NetboxDeviceCollection = get_collection( # noqa
source=netbox, name="devices"
)
await netbox_col.fetch()
netbox_col.make_keys()
print(f"{len(netbox_col)} items.", flush=True)
diff_res = diff(
source_from=ipf_col,
sync_to=netbox_col,
fields_cmp={
"model": lambda f: True # TODO: do not consider model for diff right now
},
)
if diff_res is None:
print("No changes required.")
return ipf_col
_report_proposed_changes(diff_res)
if params.get("dry_run", False) is True:
return ipf_col
updates = list()
if diff_res.missing:
updates.append(_execute_create(ipf_col, netbox_col, diff_res.missing))
if diff_res.changes:
updates.append(_execute_changes(params, ipf_col, netbox_col, diff_res.changes))
await asyncio.gather(*updates)
return ipf_col
async def ensure_sites(ipf, nb, **params):
"""
Ensure Netbox contains the sites defined in IP Fabric
Parameters
----------
ipf: IPFabric Source instance
nb: Netbox Source instance
Other Parameters
----------------
dry_run: bool
Determines dry-run mode
"""
print("Ensure Netbox contains the Sites defined in IP Fabric")
print("Fetching from IP Fabric and Netbox ... ")
ipf_col_sites: IPFabricSiteCollection = get_collection( # noqa
source=ipf, name="sites")
nb_col_sites: NetboxSiteCollection = get_collection(source=nb,
name="sites") # noqa
await asyncio.gather(ipf_col_sites.fetch(), nb_col_sites.fetch())
ipf_col_sites.make_keys()
nb_col_sites.make_keys()
print(f"IP Fabric {len(ipf_col_sites)} items.")
print(f"Netbox {len(nb_col_sites)} items.")
diff_res = diff(source_from=ipf_col_sites, sync_to=nb_col_sites)
if diff_res is None:
print("No changes required.")
return
_diff_report(diff_res=diff_res)
if params.get("dry_run", False) is True:
return
if diff_res.missing:
await _create_missing(nb_col_sites, diff_res.missing)
if diff_res.changes:
await _update_changes(nb_col_sites, diff_res.changes)
print(f"{len(ipf_col_pc)} items.")
if not len(ipf_col_pc):
return set()
hostname_set = {rec["hostname"] for rec in ipf_col_pc.inventory.values()}
print("Fetching from Netbox ... ", flush=True, end="")
await asyncio.gather(*(nb_col_pc.fetch(hostname=hostname)
for hostname in hostname_set))
nb_col_pc.make_keys()
print(f"{len(ipf_col_pc)} items.")
diff_res = diff(source_from=ipf_col_pc, sync_to=nb_col_pc)
if not diff_res:
print("No changes required.")
return hostname_set
diff_report_brief(diff_res)
if params.get("dry_run", False) is True:
return hostname_set
tasks = list()
if diff_res.missing:
tasks.append(_diff_create(nb_col_pc, diff_res.missing))
if diff_res.changes:
tasks.append(_diff_update(nb_col_pc, diff_res.changes))
async def _ensure_primary_ipaddrs(
ipf_col: IPFabricDeviceCollection, nb_col: NetboxDeviceCollection, missing: dict
):
ipf_col_ipaddrs = get_collection(source=ipf_col.source, name="ipaddrs")
ipf_col_ifaces = get_collection(source=ipf_col.source, name="interfaces")
# -------------------------------------------------------------------------
# we need to fetch all of the IPF ipaddr records so that we can bind the
# management IP address to the Netbox device record. We use the **IPF**
# collection as the basis for the missing records so that the filter values
# match. This is done to avoid any mapping changes that happended via the
# collection intake process. This code is a bit of 'leaky-abstration',
# so TODO: cleanup.
# -------------------------------------------------------------------------
await asyncio.gather(
*(
ipf_col_ipaddrs.fetch(
filters=f"and(hostname = {_item['hostname']}, ip = '{_item['loginIp']}')"
)
for _item in [ipf_col.source_record_keys[key] for key in missing.keys()]
)
)
ipf_col_ipaddrs.make_keys()
# -------------------------------------------------------------------------
# now we need to gather the IPF interface records so we have any fields that
# need to be stored into Netbox (e.g. description)
# -------------------------------------------------------------------------
await asyncio.gather(
*(
ipf_col_ifaces.fetch(
filters=f"and(hostname = {_item['hostname']}, intName = {_item['intName']})"
)
for _item in ipf_col_ipaddrs.source_record_keys.values()
)
)
ipf_col_ifaces.make_keys()
# -------------------------------------------------------------------------
# At this point we have the IPF collections for the needed 'interfaces' and
# 'ipaddrs'. We need to ensure these same entities exist in the Netbox
# collections. We will first attempt to find all the existing records in
# Netbox using the `fetch_keys` method.
# -------------------------------------------------------------------------
nb_col_ifaces = get_collection(source=nb_col.source, name="interfaces")
nb_col_ipaddrs = get_collection(source=nb_col.source, name="ipaddrs")
await nb_col_ifaces.fetch_keys(keys=ipf_col_ifaces.inventory)
await nb_col_ipaddrs.fetch_keys(keys=ipf_col_ipaddrs.inventory)
nb_col_ipaddrs.make_keys()
nb_col_ifaces.make_keys()
diff_ifaces = diff(source_from=ipf_col_ifaces, sync_to=nb_col_ifaces)
diff_ipaddrs = diff(source_from=ipf_col_ipaddrs, sync_to=nb_col_ipaddrs)
def _report_iface(item, _res: Response):
hname, iname = item["hostname"], item["interface"]
if _res.is_error:
print(f"CREATE:FAIL: interface {hname}, {iname}: {_res.text}")
return
print(f"CREATE:OK: interface {hname}, {iname}.")
nb_col_ifaces.source_records.append(_res.json())
def _report_ipaddr(item, _res: Response):
hname, iname, ipaddr = item["hostname"], item["interface"], item["ipaddr"]
ident = f"ipaddr {hname}, {iname}, {ipaddr}"
if _res.is_error:
print(f"CREATE:FAIL: {ident}: {_res.text}")
return
nb_col_ipaddrs.source_records.append(_res.json())
print(f"CREATE:OK: {ident}.")
if diff_ifaces:
await nb_col_ifaces.create_missing(
missing=diff_ifaces.missing, callback=_report_iface
)
if diff_ipaddrs:
await nb_col_ipaddrs.create_missing(
missing=diff_ipaddrs.missing, callback=_report_ipaddr
)
nb_col.make_keys()
nb_col_ifaces.make_keys()
nb_col_ipaddrs.make_keys()
# TODO: Note that I am passing the cached collections of interfaces and ipaddress
# To the device collection to avoid duplicate lookups for record
# indexes. Will give this approach some more thought.
nb_col.cache["interfaces"] = nb_col_ifaces
nb_col.cache["ipaddrs"] = nb_col_ipaddrs
async def ensure_devices(ipf, netbox, **params) -> IPFabricDeviceCollection:
"""
Ensure Netbox contains devices found IP Fabric.
Parameters
----------
ipf: IPFabric Source instance
netbox: Netbox Source instance
Other Parameters
----------------
dry_run: bool
Determines dry-run mode
devices: List[str]
List of device to use as basis for action
filters: str
The IPF device inventory filter expression to use
as basis for action.
Returns
-------
IPFabricDeviceCollection:
The IP Fabric device collection, that can be used by later processes
that need to cross reference this information.
"""
print("\nEnsure Devices.")
print("Fetching from IP Fabric ... ", flush=True, end="")
ipf_col: IPFabricDeviceCollection = get_collection( # noqa
source=ipf, name="devices"
)
filters = params["filters"]
await ipf_col.fetch(filters=filters)
ipf_col.make_keys()
print(f"{len(ipf_col)} items.", flush=True)
if not len(ipf_col.source_records):
print(f"Done. No source_records matching filter:\n\t{filters}")
return ipf_col
print("Fetching from Netbox ... ", flush=True, end="")
netbox_col: NetboxDeviceCollection = get_collection( # noqa
source=netbox, name="devices"
)
await netbox_col.fetch()
netbox_col.make_keys()
print(f"{len(netbox_col)} items.", flush=True)
diff_res = diff(
source_from=ipf_col,
sync_to=netbox_col,
fields_cmp={
"model": lambda f: True # TODO: do not consider model for diff right now
},
)
if diff_res is None:
print("No changes required.")
return ipf_col
_report_proposed_changes(netbox_col, diff_res)
if params.get("dry_run", False) is True:
return ipf_col
updates = list()
if diff_res.missing:
updates.append(_execute_create(ipf_col, netbox_col, diff_res.missing))
if diff_res.changes:
updates.append(_execute_changes(params, ipf_col, netbox_col, diff_res.changes))
await asyncio.gather(*updates)
return ipf_col