async def ensure_devices(ipf, netbox, **params) -> IPFabricDeviceCollection:
"""
Ensure Netbox contains devices found IP Fabric in given Site
"""
print("\nEnsure Devices.")
print("Fetching from IP Fabric ... ", flush=True, end="")
ipf_col: IPFabricDeviceCollection = get_collection( # noqa
source=ipf, name="devices"
)
filters = params["filters"]
await ipf_col.fetch(filters=filters)
ipf_col.make_keys()
print(f"{len(ipf_col)} items.", flush=True)
if not len(ipf_col.source_records):
print(f"Done. No source_records matching filter:\n\t{filters}")
return ipf_col
print("Fetching from Netbox ... ", flush=True, end="")
netbox_col: NetboxDeviceCollection = get_collection( # noqa
source=netbox, name="devices"
)
await netbox_col.fetch()
netbox_col.make_keys()
print(f"{len(netbox_col)} items.", flush=True)
diff_res = diff(
source_from=ipf_col,
sync_to=netbox_col,
fields_cmp={
"model": lambda f: True # TODO: do not consider model for diff right now
},
)
if diff_res is None:
print("No changes required.")
return ipf_col
_report_proposed_changes(diff_res)
if params.get("dry_run", False) is True:
return ipf_col
updates = list()
if diff_res.missing:
updates.append(_execute_create(ipf_col, netbox_col, diff_res.missing))
if diff_res.changes:
updates.append(_execute_changes(params, ipf_col, netbox_col, diff_res.changes))
await asyncio.gather(*updates)
return ipf_col
async def ensure_lags(ipf, nb, **params) -> Set[str]:
print("\nEnsure Device LAG member interfaces.")
ipf_col_pc: IPFabricPortChannelCollection = get_collection( # noqa
source=ipf, name="portchans")
nb_col_pc: NetboxPortChanCollection = get_collection( # noqa
source=nb, name="portchans")
print("Fetching from IP Fabric ... ", flush=True, end="")
if (filters := params.get("filters")) is not None:
await ipf_col_pc.fetch(filters=filters)
async def create_missing(self,
missing: Dict,
callback: Optional[CollectionCallback] = None):
# missing items means that the existing interface does not have any
# associated LAG. We need to patch the interface record with the
# LAG id.
api: NetboxClient = self.source.client
# we first need to retrieve all of the interface records
col_ifaces = get_collection(source=self.source, name="interfaces")
async for _ in igather(
(col_ifaces.fetch(hostname=item["hostname"],
name=item["interface"])
for item in missing.values()),
limit=100,
):
pass
col_ifaces.make_keys()
def _patch(key, item):
if_rec = col_ifaces.source_record_keys[key]
lag_key = (item["hostname"], item["portchan"])
lag_rec = self.cache[self]["lag_recs"][lag_key]
return api.patch(_INTFS_URL + f"{if_rec['id']}/",
json=dict(lag=lag_rec["id"]))
await self.source.update(missing, callback=callback, creator=_patch)
async def update_changes(self,
changes: Dict,
callback: Optional[CollectionCallback] = None):
# we first need to retrieve all of the interface records
col_ifaces = get_collection(source=self.source, name="interfaces")
async for _ in igather(
(col_ifaces.fetch(
hostname=item.fingerprint["hostname"],
name=item.fingerprint["interface"],
) for item in changes.values()),
limit=100,
):
pass
col_ifaces.make_keys()
api: NetboxClient = self.source.client
def _patch(_key, _ch_fields):
if_rec = col_ifaces.source_record_keys[_key]
col_fields = self.inventory[_key]
lag_key = (col_fields["hostname"], _ch_fields["portchan"])
lag_rec = self.cache[self]["lag_recs"][lag_key]
return api.patch(_INTFS_URL + f"{if_rec['id']}/",
json=dict(lag=lag_rec["id"]))
await self.source.update(changes, callback=callback, creator=_patch)
async def ensure_sites(ipf, nb, **params):
"""
Ensure Netbox contains the sites defined in IP Fabric
Parameters
----------
ipf: IPFabric Source instance
nb: Netbox Source instance
Other Parameters
----------------
dry_run: bool
Determines dry-run mode
"""
print("Ensure Netbox contains the Sites defined in IP Fabric")
print("Fetching from IP Fabric and Netbox ... ")
ipf_col_sites: IPFabricSiteCollection = get_collection( # noqa
source=ipf, name="sites")
nb_col_sites: NetboxSiteCollection = get_collection(source=nb,
name="sites") # noqa
await asyncio.gather(ipf_col_sites.fetch(), nb_col_sites.fetch())
ipf_col_sites.make_keys()
nb_col_sites.make_keys()
print(f"IP Fabric {len(ipf_col_sites)} items.")
print(f"Netbox {len(nb_col_sites)} items.")
diff_res = diff(source_from=ipf_col_sites, sync_to=nb_col_sites)
if diff_res is None:
print("No changes required.")
return
_diff_report(diff_res=diff_res)
if params.get("dry_run", False) is True:
return
if diff_res.missing:
await _create_missing(nb_col_sites, diff_res.missing)
if diff_res.changes:
await _update_changes(nb_col_sites, diff_res.changes)
async def ensure_ipaddrs(ipf, nb, **params) -> IPFabricIPAddrCollection:
print("\nEnsure IP Address.")
print("Fetching from IP Fabric ... ", flush=True, end="")
ipf_col: IPFabricIPAddrCollection = get_collection( # noqa
source=ipf, name="ipaddrs")
if (filters := params.get("filters")) is not None:
await ipf_col.fetch(filters=filters)
async def ensure_interfaces(ipf, nb, **params) -> IPFabricInterfaceCollection:
print("\nEnsure Device Interfaces.")
# -------------------------------------------------------------------------
# Fetch from IP Fabric with the User provided filter expression.
# -------------------------------------------------------------------------
print("Fetching from IP Fabric ... ", flush=True, end="")
ipf_col: IPFabricInterfaceCollection = get_collection( # noqa
source=ipf, name="interfaces"
)
if (filters := params.get("filters")) is not None:
await ipf_col.fetch(filters=filters)
async def ensure_interfaces(ipf, nb, **params) -> List[str]:
"""
Ensure Netbox contains devices interfaces found IP Fabric.
Parameters
----------
ipf: IPFabric Source instance
nb: Netbox Source instance
Other Parameters
----------------
dry_run: bool
Determines dry-run mode
devices: List[str]
List of device to use as basis for action
filters: str
The IPF device inventory filter expression to use
as basis for action.
Returns
-------
List[str]
The list of IPF device hostnames found in the IPF collection. Can
be used as a basis for other collection activities.
"""
print("\nEnsure Device Interfaces.")
# -------------------------------------------------------------------------
# Fetch from IP Fabric with the User provided filter expression.
# -------------------------------------------------------------------------
print("Fetching from IP Fabric ... ", flush=True, end="")
ipf_col: IPFabricInterfaceCollection = get_collection( # noqa
source=ipf, name="interfaces"
)
if (filters := params.get("filters")) is not None:
await ipf_col.fetch(filters=filters)
async def remove_extra(self,
extras: Dict,
callback: Optional[CollectionCallback] = None):
api: NetboxClient = self.source.client
# we first need to retrieve all of the interface records
col_ifaces = get_collection(source=self.source, name="interfaces")
async for _ in igather(
(col_ifaces.fetch(hostname=item["hostname"],
name=item["interface"])
for item in extras.values()),
limit=100,
):
pass
col_ifaces.make_keys()
def _patch(key, _fields):
if_rec = col_ifaces.source_record_keys[key]
return api.patch(_INTFS_URL + f"{if_rec['id']}/",
json=dict(lag=None))
await self.source.update(extras, callback=callback, creator=_patch)
print(f"{len(ipf_col)} items.", flush=True)
if not len(ipf_col):
return []
# create the IPF hostname specific device list for return purposes.
ipf_device_list = [rec["hostname"] for rec in ipf_col.source_records]
# -------------------------------------------------------------------------
# Need to fetch interfaces from Netbox on a per-device basis.
# -------------------------------------------------------------------------
print("Fetching from Netbox ... ", flush=True, end="")
nb_col: NetboxInterfaceCollection = get_collection( # noqa
source=nb, name="interfaces"
)
col_device_list = {rec["hostname"] for rec in ipf_col.inventory.values()}
print(f"{len(col_device_list)} devices ... ", flush=True, end="")
nb.client.timeout = 120
await asyncio.gather(
*(nb_col.fetch(hostname=hostname) for hostname in col_device_list)
)
nb_col.make_keys()
print(f"{len(nb_col)} items.", flush=True)
# -------------------------------------------------------------------------
# check for differences and process accordingly.
async def _ensure_primary_ipaddrs(
ipf_col: IPFabricDeviceCollection, nb_col: NetboxDeviceCollection, missing: dict
):
ipf_col_ipaddrs = get_collection(source=ipf_col.source, name="ipaddrs")
ipf_col_ifaces = get_collection(source=ipf_col.source, name="interfaces")
# -------------------------------------------------------------------------
# we need to fetch all of the IPF ipaddr records so that we can bind the
# management IP address to the Netbox device record. We use the **IPF**
# collection as the basis for the missing records so that the filter values
# match. This is done to avoid any mapping changes that happended via the
# collection intake process. This code is a bit of 'leaky-abstration',
# so TODO: cleanup.
# -------------------------------------------------------------------------
await asyncio.gather(
*(
ipf_col_ipaddrs.fetch(
filters=f"and(hostname = {_item['hostname']}, ip = '{_item['loginIp']}')"
)
for _item in [ipf_col.source_record_keys[key] for key in missing.keys()]
)
)
ipf_col_ipaddrs.make_keys()
# -------------------------------------------------------------------------
# now we need to gather the IPF interface records so we have any fields that
# need to be stored into Netbox (e.g. description)
# -------------------------------------------------------------------------
await asyncio.gather(
*(
ipf_col_ifaces.fetch(
filters=f"and(hostname = {_item['hostname']}, intName = {_item['intName']})"
)
for _item in ipf_col_ipaddrs.source_record_keys.values()
)
)
ipf_col_ifaces.make_keys()
# -------------------------------------------------------------------------
# At this point we have the IPF collections for the needed 'interfaces' and
# 'ipaddrs'. We need to ensure these same entities exist in the Netbox
# collections. We will first attempt to find all the existing records in
# Netbox using the `fetch_keys` method.
# -------------------------------------------------------------------------
nb_col_ifaces = get_collection(source=nb_col.source, name="interfaces")
nb_col_ipaddrs = get_collection(source=nb_col.source, name="ipaddrs")
await nb_col_ifaces.fetch_keys(keys=ipf_col_ifaces.inventory)
await nb_col_ipaddrs.fetch_keys(keys=ipf_col_ipaddrs.inventory)
nb_col_ipaddrs.make_keys()
nb_col_ifaces.make_keys()
diff_ifaces = diff(source_from=ipf_col_ifaces, sync_to=nb_col_ifaces)
diff_ipaddrs = diff(source_from=ipf_col_ipaddrs, sync_to=nb_col_ipaddrs)
def _report_iface(item, _res: Response):
hname, iname = item["hostname"], item["interface"]
if _res.is_error:
print(f"CREATE:FAIL: interface {hname}, {iname}: {_res.text}")
return
print(f"CREATE:OK: interface {hname}, {iname}.")
nb_col_ifaces.source_records.append(_res.json())
def _report_ipaddr(item, _res: Response):
hname, iname, ipaddr = item["hostname"], item["interface"], item["ipaddr"]
ident = f"ipaddr {hname}, {iname}, {ipaddr}"
if _res.is_error:
print(f"CREATE:FAIL: {ident}: {_res.text}")
return
nb_col_ipaddrs.source_records.append(_res.json())
print(f"CREATE:OK: {ident}.")
if diff_ifaces:
await nb_col_ifaces.create_missing(
missing=diff_ifaces.missing, callback=_report_iface
)
if diff_ipaddrs:
await nb_col_ipaddrs.create_missing(
missing=diff_ipaddrs.missing, callback=_report_ipaddr
)
nb_col.make_keys()
nb_col_ifaces.make_keys()
nb_col_ipaddrs.make_keys()
# TODO: Note that I am passing the cached collections of interfaces and ipaddress
# To the device collection to avoid duplicate lookups for record
# indexes. Will give this approach some more thought.
nb_col.cache["interfaces"] = nb_col_ifaces
nb_col.cache["ipaddrs"] = nb_col_ipaddrs
ipf_col.make_keys()
if not len(ipf_col.source_records):
print(f"0 items matching filter: `{filters}`.")
return ipf_col
print(f"{len(ipf_col)} items.")
# -------------------------------------------------------------------------
# Need to fetch from Netbox on a per-device basis.
# -------------------------------------------------------------------------
print("Fetching from Netbox ... ", flush=True, end="")
nb_col: NetboxIPAddrCollection = get_collection(source=nb,
name="ipaddrs") # noqa
device_list = {rec["hostname"] for rec in ipf_col.inventory.values()}
print(f"{len(device_list)} devices ... ", flush=True, end="")
nb.client.timeout = 120
await asyncio.gather(*(nb_col.fetch(hostname=hostname)
for hostname in device_list))
nb_col.make_keys()
print(f"{len(nb_col)} items.", flush=True)
# -------------------------------------------------------------------------
# check for differences and process accordingly.
# -------------------------------------------------------------------------
async def ensure_devices(ipf, netbox, **params) -> IPFabricDeviceCollection:
"""
Ensure Netbox contains devices found IP Fabric.
Parameters
----------
ipf: IPFabric Source instance
netbox: Netbox Source instance
Other Parameters
----------------
dry_run: bool
Determines dry-run mode
devices: List[str]
List of device to use as basis for action
filters: str
The IPF device inventory filter expression to use
as basis for action.
Returns
-------
IPFabricDeviceCollection:
The IP Fabric device collection, that can be used by later processes
that need to cross reference this information.
"""
print("\nEnsure Devices.")
print("Fetching from IP Fabric ... ", flush=True, end="")
ipf_col: IPFabricDeviceCollection = get_collection( # noqa
source=ipf, name="devices"
)
filters = params["filters"]
await ipf_col.fetch(filters=filters)
ipf_col.make_keys()
print(f"{len(ipf_col)} items.", flush=True)
if not len(ipf_col.source_records):
print(f"Done. No source_records matching filter:\n\t{filters}")
return ipf_col
print("Fetching from Netbox ... ", flush=True, end="")
netbox_col: NetboxDeviceCollection = get_collection( # noqa
source=netbox, name="devices"
)
await netbox_col.fetch()
netbox_col.make_keys()
print(f"{len(netbox_col)} items.", flush=True)
diff_res = diff(
source_from=ipf_col,
sync_to=netbox_col,
fields_cmp={
"model": lambda f: True # TODO: do not consider model for diff right now
},
)
if diff_res is None:
print("No changes required.")
return ipf_col
_report_proposed_changes(netbox_col, diff_res)
if params.get("dry_run", False) is True:
return ipf_col
updates = list()
if diff_res.missing:
updates.append(_execute_create(ipf_col, netbox_col, diff_res.missing))
if diff_res.changes:
updates.append(_execute_changes(params, ipf_col, netbox_col, diff_res.changes))
await asyncio.gather(*updates)
return ipf_col
import os
import asyncio
from ipf_netbox.source import get_source
from ipf_netbox.collection import get_collection
from ipf_netbox.config import load_config_file
load_config_file(filepath=open(os.getenv("IPF_NETBOX_CONFIG")))
ipf_col_pc = get_collection(source=get_source("ipfabric"), name="portchans")
nb_col_pc = get_collection(source=get_source("netbox"), name="portchans")
async def run(**params):
async with ipf_col_pc.source.client, nb_col_pc.source.client:
await ipf_col_pc.fetch(**params)
ipf_col_pc.make_keys()
hostname_list = {rec["hostname"] for rec in ipf_col_pc.inventory.values()}
await asyncio.gather(
*(nb_col_pc.fetch(hostname=hostname) for hostname in hostname_list)
)
nb_col_pc.make_keys()
