def load(self) -> Inventory:
if self.defaults_dict:
defaults = _get_defaults(self.defaults_dict)
else:
defaults = Defaults()
hosts = Hosts()
hosts_dict = self.host_dict
for n, h in hosts_dict.items():
hosts[n] = _get_inventory_element(Host, h, n, defaults)
groups = Groups()
if self.group_dict:
groups_dict = self.group_dict
for n, g in groups_dict.items():
groups[n] = _get_inventory_element(Group, g, n, defaults)
for h in hosts.values():
h.groups = ParentGroups([groups[g] for g in h.groups])
for g in groups.values():
g.groups = ParentGroups([groups[g] for g in g.groups])
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self) -> Inventory:
yml = ruamel.yaml.YAML(typ="safe")
if self.defaults_file.exists():
with open(self.defaults_file, "r") as f:
defaults_dict = yml.load(f) or {}
defaults = _get_defaults(defaults_dict)
else:
defaults = Defaults()
hosts = Hosts()
with open(self.host_file, "r") as f:
hosts_dict = yml.load(f)
for n, h in hosts_dict.items():
hosts[n] = _get_inventory_element(Host, h, n, defaults)
groups = Groups()
if self.group_file.exists():
with open(self.group_file, "r") as f:
groups_dict = yml.load(f) or {}
for n, g in groups_dict.items():
groups[n] = _get_inventory_element(Group, g, n, defaults)
for h in hosts.values():
h.groups = ParentGroups([groups[g] for g in h.groups])
for g in groups.values():
g.groups = ParentGroups([groups[g] for g in g.groups])
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self) -> Inventory:
"""Standard Nornir 3 load method boilerplate."""
if not self.credentials_params:
self.credentials_params = {}
# Initialize QuerySet
if not self.queryset:
self.queryset = Device.objects.all()
if self.filters:
self.queryset = self.queryset.filter(**self.filters)
if not self.params:
self.params = {}
self.queryset = self.queryset.select_related(
"device_role",
"device_type",
"device_type__manufacturer",
"site",
"platform",
"tenant",
)
# Initialize Hosts and Groups vars
hosts = Hosts()
defaults = Defaults(data=self.defaults)
groups = Groups()
if self.credentials_params:
cred = self.cred_class(params=self.credentials_params)
else:
cred = self.cred_class()
# Create all hosts
for device in self.queryset:
host = self.create_host(device=device,
cred=cred,
params=self.params)
hosts[device.name] = _set_host(data=host["data"],
name=host["name"],
groups=host["groups"],
host=host,
defaults=defaults)
# Initialize all groups if they don't already exist
for group in hosts[device.name].groups:
if group not in groups.keys():
groups[group] = Group(name=group, defaults=defaults)
for _host in hosts.values():
_host.groups = ParentGroups(
[groups[_group] for _group in _host.groups])
for _group in groups.values():
_group.groups = ParentGroups(
[groups[_group] for _group in _group.groups])
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self) -> Inventory:
hosts = Hosts()
groups = Groups()
defaults = Defaults()
hosts_redis = check_redis("nornir_hosts")
groups_redis = check_redis("nornir_groups")
if len(hosts_redis) > 0 and len(groups_redis) > 0:
print(f"Found cached info in Redis and will used that")
serialized_groups = groups_redis
serialized_hosts = hosts_redis
else:
serialized_groups, lnetd_groups = _lnetd_get_groups(self.lnetd_db)
serialized_hosts = _lnetd_get_hosts(self.lnetd_db, lnetd_groups)
for h in serialized_hosts.values():
h.groups = ParentGroups(
[serialized_groups[g] for g in h.groups])
update_redis("nornir_hosts", serialized_hosts)
update_redis("nornir_groups", serialized_groups)
return Inventory(hosts=serialized_hosts,
groups=serialized_groups,
defaults=defaults)
def load(self) -> Inventory:
nb_devices: List[Dict[str, Any]] = []
nb_devices = self._get_resources(
url=f"{self.nb_url}/api/dcim/devices/?limit=0",
params=self.filter_parameters,
)
if self.include_vms:
nb_devices.extend(
self._get_resources(
url=f"{self.nb_url}/api/virtualization/virtual-machines/?limit=0",
params=self.filter_parameters,
)
)
hosts = Hosts()
groups = Groups()
defaults = Defaults()
for device in nb_devices:
serialized_device: Dict[Any, Any] = {}
serialized_device["data"] = device
if self.flatten_custom_fields:
for cf, value in device["custom_fields"].items():
serialized_device["data"][cf] = value
serialized_device["data"].pop("custom_fields")
hostname = None
if device.get("primary_ip"):
hostname = device.get("primary_ip", {}).get("address", "").split("/")[0]
else:
if device.get("name") is not None:
hostname = device["name"]
serialized_device["hostname"] = hostname
if isinstance(device["platform"], dict) and self.use_platform_slug:
platform = device["platform"].get("slug")
elif isinstance(device["platform"], dict):
platform = device["platform"].get("name")
else:
platform = device["platform"]
serialized_device["platform"] = platform
name = serialized_device["data"].get("name") or str(
serialized_device["data"].get("id")
)
hosts[name] = _get_inventory_element(
Host, serialized_device, name, defaults
)
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self):
return Inventory(
hosts=Hosts({
"h1": Host("h1"),
"h2": Host("h2"),
"h3": Host("h3")
}),
groups=Groups({"g1": Group("g1")}),
defaults=Defaults(),
)
def load(self) -> Inventory:
defaults = _get_defaults(self.defaults_dict)
hosts = Hosts()
hosts_dict = _get_devices_excel(self._excel_file)
for n, h in hosts_dict.items():
hosts[n] = _get_inventory_element(Host, h, n, defaults)
groups = Groups()
for n, g in self.groups_dict.items():
groups[n] = _get_inventory_element(Group, g, n, defaults)
for h in hosts.values():
h.groups = ParentGroups([groups[g] for g in h.groups])
for g in groups.values():
g.groups = ParentGroups([groups[g] for g in g.groups])
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self) -> Inventory:
url = f"{self.nb_url}/api/dcim/devices/?limit=0"
nb_devices: List[Dict[str, Any]] = []
while url:
r = self.session.get(url, params=self.filter_parameters)
if not r.status_code == 200:
raise ValueError(
f"Failed to get devices from NetBox instance {self.nb_url}"
)
resp = r.json()
nb_devices.extend(resp.get("results"))
url = resp.get("next")
hosts = Hosts()
groups = Groups()
defaults = Defaults()
for device in nb_devices:
serialized_device: Dict[Any, Any] = {}
serialized_device["data"] = device
if self.flatten_custom_fields:
for cf, value in device["custom_fields"].items():
serialized_device["data"][cf] = value
serialized_device["data"].pop("custom_fields")
hostname = None
if device.get("primary_ip"):
hostname = device.get("primary_ip", {}).get("address",
"").split("/")[0]
else:
if device.get("name") is not None:
hostname = device["name"]
serialized_device["hostname"] = hostname
platform = (device["platform"]["name"] if isinstance(
device["platform"], dict) else device["platform"])
serialized_device["platform"] = platform
name = serialized_device["data"].get("name") or str(
serialized_device["data"].get("id"))
hosts[name] = _get_inventory_element(Host, serialized_device, name,
defaults)
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self) -> Inventory:
if self.defaults_file.exists():
with open(self.defaults_file, "r") as f:
try:
defaults_dict = json.load(f)
except JSONDecodeError:
defaults_dict = {}
defaults = _get_defaults(defaults_dict)
else:
defaults = Defaults()
hosts = Hosts()
with open(self.host_file, "r") as f:
hosts_dict = json.load(f)
for n, h in hosts_dict.items():
hosts[n] = _get_inventory_element(Host, h, n, defaults)
groups = Groups()
if self.group_file.exists():
with open(self.group_file, "r") as f:
try:
groups_dict = json.load(f)
except JSONDecodeError:
groups_dict = {}
for n, g in groups_dict.items():
groups[n] = _get_inventory_element(Group, g, n, defaults)
for h in hosts.values():
h.groups = ParentGroups([groups[g] for g in h.groups])
for g in groups.values():
g.groups = ParentGroups([groups[g] for g in g.groups])
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def _lnetd_get_groups(lnetd_db):
conn = sqlite3.connect(lnetd_db)
sql_groups = "select * from tag join tags on tag.id = tags.tag_id"
df_groups = pd.read_sql(sql_groups, conn)
df_groups.replace("", float("NaN"),
inplace=True) # replace empty tags with NaN
df_groups.dropna(inplace=True) # drop NaN values
lnetd_groups = df_groups.to_dict(orient="records")
groups_initial = {}
groups = Groups()
defaults = Defaults()
for gr in lnetd_groups:
group: GroupDict = {"data": {}}
group["data"]["role"] = gr["name"]
groups_initial[gr["name"]] = group
for group_name, group_data in groups_initial.items():
groups[group_name] = _get_inventory_element(Group, group_data,
group_name, defaults)
return groups, lnetd_groups
def load(self) -> Inventory:
# we retrieve the data from the inventory passing the options we saved
# in he constructor
data = self.conn.get(self.filter_sites, self.filter_dev_types)
# we create placeholder for the hosts and for the groups
hosts = Hosts()
groups = Groups()
# the inventory returned the hosts by datacenter so we iterate over them
for dc_name, dc_data in data.items():
# we are going to start bt creating a group per DC
groups[dc_name] = Group(dc_name)
# now we process the dc data we got
hosts_in_dc = process_dc_data(groups[dc_name], dc_data)
# we add the hosts in the dc to the main hosts object
hosts.update(hosts_in_dc)
# we populate the inventory and return it
# note our inventory doesn't support defaults so we just return
# and empty object
return Inventory(hosts=hosts, groups=groups, defaults=Defaults())
def load(self) -> Inventory:
yml = ruamel.yaml.YAML(typ="safe")
platforms: List[Dict[str, Any]] = []
if self.use_platform_napalm_driver:
platforms = self._get_resources(
url=f"{self.nb_url}/api/dcim/platforms/?limit=0", params={})
nb_devices: List[Dict[str, Any]] = []
nb_devices = self._get_resources(
url=f"{self.nb_url}/api/dcim/devices/?limit=0",
params=self.filter_parameters,
)
if self.include_vms:
nb_devices.extend(
self._get_resources(
url=
f"{self.nb_url}/api/virtualization/virtual-machines/?limit=0",
params=self.filter_parameters,
))
hosts = Hosts()
groups = Groups()
defaults = Defaults()
if self.defaults_file.exists():
with self.defaults_file.open("r") as f:
defaults_dict = yml.load(f) or {}
defaults = _get_defaults(defaults_dict)
else:
defaults = Defaults()
if self.group_file.exists():
with self.group_file.open("r") as f:
groups_dict = yml.load(f) or {}
for n, g in groups_dict.items():
groups[n] = _get_inventory_element(Group, g, n, defaults)
for g in groups.values():
g.groups = ParentGroups([groups[g] for g in g.groups])
for device in nb_devices:
serialized_device: Dict[Any, Any] = {}
serialized_device["data"] = device
if self.flatten_custom_fields:
for cf, value in device["custom_fields"].items():
serialized_device["data"][cf] = value
serialized_device["data"].pop("custom_fields")
hostname = None
if device.get("primary_ip"):
hostname = device.get("primary_ip", {}).get("address",
"").split("/")[0]
else:
if device.get("name") is not None:
hostname = device["name"]
serialized_device["hostname"] = hostname
if isinstance(device["platform"], dict) and self.use_platform_slug:
platform = device["platform"].get("slug")
elif (isinstance(device["platform"], dict)
and self.use_platform_napalm_driver):
platform = [
platform for platform in platforms
if device["platform"]["slug"] == platform["slug"]
][0]["napalm_driver"]
elif isinstance(device["platform"], dict):
platform = device["platform"].get("name")
else:
platform = device["platform"]
serialized_device["platform"] = platform
name = serialized_device["data"].get("name") or str(
serialized_device["data"].get("id"))
hosts[name] = _get_inventory_element(Host, serialized_device, name,
defaults)
groups_extracted = self._extract_device_groups(device)
for group in groups_extracted:
if group not in groups.keys():
groups[group] = _get_inventory_element(
Group, {}, group, defaults)
hosts[name].groups = ParentGroups(
[groups[g] for g in groups_extracted])
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self) -> Inventory:
url = f"{self.base_url}/api/dcim/devices/?limit=0"
nb_devices: List[Dict[str, Any]] = []
while url:
r = self.session.get(url, params=self.filter_parameters)
if not r.status_code == 200:
raise ValueError(
f"Failed to get devices from NetBox instance {self.base_url}"
)
resp = r.json()
nb_devices.extend(resp.get("results"))
url = resp.get("next")
hosts = Hosts()
groups = Groups()
defaults = Defaults()
for device in nb_devices:
serialized_device: Dict[Any, Any] = {}
serialized_device["data"] = {}
serialized_device["data"]["serial"] = device.get("serial")
serialized_device["data"]["vendor"] = (device.get(
"device_type", {}).get("manufacturer", {}).get("name"))
serialized_device["data"]["asset_tag"] = device.get("asset_tag")
if self.flatten_custom_fields:
for key, value in device.get("custom_fields", {}).items():
serialized_device["data"][key] = value
else:
serialized_device["data"]["custom_fields"] = device.get(
"custom_fields", {})
if self.use_slugs:
serialized_device["data"]["site"] = device.get("site",
{}).get("slug")
serialized_device["data"]["role"] = device.get(
"device_role", {}).get("slug")
serialized_device["data"]["model"] = device.get(
"device_type", {}).get("slug")
serialized_device["platform"] = (
device["platform"]["slug"] if isinstance(
device["platform"], dict) else device["platform"])
else:
serialized_device["data"]["site"] = device.get("site",
{}).get("name")
serialized_device["data"]["role"] = device.get("device_role")
serialized_device["data"]["model"] = device.get("device_type")
serialized_device["platform"] = (
device["platform"]["name"] if isinstance(
device["platform"], dict) else device["platform"])
serialized_device["hostname"] = None
if device.get("primary_ip"):
serialized_device["hostname"] = (device.get(
"primary_ip", {}).get("address", "").split("/")[0])
else:
if device.get("name") is not None:
serialized_device["hostname"] = device["name"]
name = device.get("name") or str(device.get("id"))
hosts[name] = _get_inventory_element(Host, serialized_device, name,
defaults)
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def inventory_from_yaml():
dir_path = os.path.dirname(os.path.realpath(__file__))
yml = ruamel.yaml.YAML(typ="safe")
def get_connection_options(data):
cp = {}
for cn, c in data.items():
cp[cn] = ConnectionOptions(
hostname=c.get("hostname"),
port=c.get("port"),
username=c.get("username"),
password=c.get("password"),
platform=c.get("platform"),
extras=c.get("extras"),
)
return cp
def get_defaults():
defaults_file = f"{dir_path}/inventory_data/defaults.yaml"
with open(defaults_file, "r") as f:
defaults_dict = yml.load(f)
defaults = Defaults(
hostname=defaults_dict.get("hostname"),
port=defaults_dict.get("port"),
username=defaults_dict.get("username"),
password=defaults_dict.get("password"),
platform=defaults_dict.get("platform"),
data=defaults_dict.get("data"),
connection_options=get_connection_options(
defaults_dict.get("connection_options", {})),
)
return defaults
def get_inventory_element(typ, data, name, defaults):
return typ(
name=name,
hostname=data.get("hostname"),
port=data.get("port"),
username=data.get("username"),
password=data.get("password"),
platform=data.get("platform"),
data=data.get("data"),
groups=data.get(
"groups"
), # this is a hack, we will convert it later to the correct type
defaults=defaults,
connection_options=get_connection_options(
data.get("connection_options", {})),
)
host_file = f"{dir_path}/inventory_data/hosts.yaml"
group_file = f"{dir_path}/inventory_data/groups.yaml"
defaults = get_defaults()
hosts = Hosts()
with open(host_file, "r") as f:
hosts_dict = yml.load(f)
for n, h in hosts_dict.items():
hosts[n] = get_inventory_element(Host, h, n, defaults)
groups = Groups()
with open(group_file, "r") as f:
groups_dict = yml.load(f)
for n, g in groups_dict.items():
groups[n] = get_inventory_element(Group, g, n, defaults)
for h in hosts.values():
h.groups = ParentGroups([groups[g] for g in h.groups])
for g in groups.values():
g.groups = ParentGroups([groups[g] for g in g.groups])
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self):
"""Load inventory by fetching devices from netbox."""
if self.filter_parameters:
devices: List[pynetbox.modules.dcim.
Devices] = self.session.dcim.devices.filter(
**self.filter_parameters)
else:
devices: List[pynetbox.modules.dcim.
Devices] = self.session.dcim.devices.all()
# fetch all platforms from Netbox and build mapping: platform: napalm_driver
platforms = self.session.dcim.platforms.all()
platforms_mapping = {
platform.slug: platform.napalm_driver
for platform in platforms if platform.napalm_driver
}
hosts = Hosts()
groups = Groups()
defaults = Defaults()
for dev in devices:
# Netbox allows devices to be unnamed, but the Nornir model does not allow this
# If a device is unnamed we will set the name to the id of the device in netbox
dev_name = dev.name or dev.id
host = NetworkImporterHost(name=dev_name,
connection_options=ConnectionOptions())
# Only add virtual chassis master as inventory element
if dev.virtual_chassis and dev.virtual_chassis.master:
if dev.id != dev.virtual_chassis.master.id:
continue
host.data["virtual_chassis"] = True
else:
host.data["virtual_chassis"] = False
# If supported_platforms is provided
# skip all devices that do not match the list of supported platforms
# TODO need to see if we can filter when doing the query directly
if self.supported_platforms:
if not dev.platform:
continue
if dev.platform.slug not in self.supported_platforms:
continue
# Add value for IP address
if self.settings.use_primary_ip and dev.primary_ip:
host.hostname = dev.primary_ip.address.split("/")[0]
elif self.settings.use_primary_ip and not dev.primary_ip:
host.is_reachable = False
host.not_reachable_reason = "primary ip not defined in Netbox"
elif not self.settings.use_primary_ip and self.settings.fqdn:
host.hostname = f"{dev.name}.{self.settings.fqdn}"
elif not self.settings.use_primary_ip:
host.hostname = dev.name
else:
host.hostname = dev_name
host.site_name = dev.site.slug
host.data["serial"] = dev.serial
host.data["vendor"] = dev.device_type.manufacturer.slug
host.data["asset_tag"] = dev.asset_tag
host.data["custom_fields"] = dev.custom_fields
host.data["site_id"] = dev.site.id
host.data["device_id"] = dev.id
host.data["role"] = dev.device_role.slug
host.data["model"] = dev.device_type.slug
# Attempt to add 'platform' based of value in 'slug'
if dev.platform and dev.platform.slug in platforms_mapping:
host.connection_options = {
"napalm":
ConnectionOptions(
platform=platforms_mapping[dev.platform.slug])
}
if dev.platform:
host.platform = dev.platform.slug
else:
host.platform = None
host.groups = ParentGroups([self.global_group])
if dev.site.slug not in groups.keys():
groups[dev.site.slug] = {}
if dev.device_role.slug not in groups.keys():
groups[dev.device_role.slug] = {}
if host.hostname and host.platform:
host.is_reachable = True
# Assign temporary dict to outer dict
hosts[dev_name] = host
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self) -> Inventory:
defaults = Defaults(
connection_options={
"napalm": ConnectionOptions(extras={
"optional_args": {
# args to eAPI HttpsEapiConnection for EOS
"enforce_verification": True,
"context": ssl_context
}
})
}
)
insecure_device_states = [
DeviceState.INIT,
DeviceState.DHCP_BOOT,
DeviceState.PRE_CONFIGURED,
DeviceState.DISCOVERED
]
insecure_connection_options = {
"napalm": ConnectionOptions(extras={
"optional_args": {"enforce_verification": False}
})
}
groups = Groups()
for device_type in list(DeviceType.__members__):
group_name = 'T_'+device_type
groups[group_name] = Group(name=group_name, defaults=defaults)
for device_state in list(DeviceState.__members__):
username, password = self._get_credentials(device_state)
group_name = 'S_'+device_state
groups[group_name] = Group(
name=group_name, username=username, password=password, defaults=defaults)
for group_name in get_groups():
groups[group_name] = Group(name=group_name, defaults=defaults)
hosts = Hosts()
with cnaas_nms.db.session.sqla_session() as session:
instance: Device
for instance in session.query(Device):
hostname = self._get_management_ip(instance.management_ip,
instance.dhcp_ip)
port = None
if instance.port and isinstance(instance.port, int):
port = instance.port
host_groups = [
'T_' + instance.device_type.name,
'S_' + instance.state.name
]
for member_group in get_groups(instance.hostname):
host_groups.append(member_group)
if instance.state in insecure_device_states:
host_connection_options = insecure_connection_options
else:
host_connection_options = None
hosts[instance.hostname] = Host(
name=instance.hostname,
hostname=hostname,
platform=instance.platform,
groups=ParentGroups(groups[g] for g in host_groups),
port=port,
data={
'synchronized': instance.synchronized,
'managed': (True if instance.state == DeviceState.MANAGED else False)
},
connection_options=host_connection_options,
defaults=defaults
)
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self) -> Inventory:
"""
Load inventory
"""
url = f"{self.ipf_url}/api/v1/tables/inventory/devices"
if self.ipf_token:
headers = {
"Content-type": "application/json",
"X-API-Token": self.ipf_token,
}
else:
credentials = b64encode(
f"{self.ipf_user}:{self.ipf_password}".encode("utf-8")).decode(
"utf-8")
headers = {
"Content-type": "application/json",
"Authorization": f"Basic {credentials}",
}
data = {
"columns": [
"loginIp",
"family",
"hostname",
"platform",
"loginType",
"sn",
"siteName",
"vendor",
"version",
],
"filters": {},
"snapshot":
self.ipf_snapshot,
}
try: # get device inventory from IP Fabric
deviceInventory = requests.post(
url,
data=json.dumps(data),
headers=headers,
verify=self.ssl_verify,
)
except:
raise ValueError(
f"Failed to get devices from IP Fabric {self.ipf_url}")
if not deviceInventory.status_code == 200:
raise ValueError(
f"Failed to get devices from IP Fabric {self.ipf_url}")
ipf_devices = json.loads(deviceInventory.content).get("data")
groups = Groups()
username = self.default.get("username", None)
password = self.default.get("password", None)
defaults = Defaults(username=username, password=password)
serialized_hosts = {}
for device in ipf_devices:
serialized_hosts[device.get("loginIp")] = _get_inventory_element(
Host, device, defaults)
return Inventory(hosts=serialized_hosts,
groups=groups,
defaults=defaults)