def load(self) -> Inventory:
yml = ruamel.yaml.YAML(typ="safe")
if self.defaults_file.exists():
with open(self.defaults_file, "r") as f:
defaults_dict = yml.load(f) or {}
defaults = _get_defaults(defaults_dict)
else:
defaults = Defaults()
hosts = Hosts()
with open(self.host_file, "r") as f:
hosts_dict = yml.load(f)
for n, h in hosts_dict.items():
hosts[n] = _get_inventory_element(Host, h, n, defaults)
groups = Groups()
if self.group_file.exists():
with open(self.group_file, "r") as f:
groups_dict = yml.load(f) or {}
for n, g in groups_dict.items():
groups[n] = _get_inventory_element(Group, g, n, defaults)
for h in hosts.values():
h.groups = ParentGroups([groups[g] for g in h.groups])
for g in groups.values():
g.groups = ParentGroups([groups[g] for g in g.groups])
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self) -> Inventory:
if self.defaults_dict:
defaults = _get_defaults(self.defaults_dict)
else:
defaults = Defaults()
hosts = Hosts()
hosts_dict = self.host_dict
for n, h in hosts_dict.items():
hosts[n] = _get_inventory_element(Host, h, n, defaults)
groups = Groups()
if self.group_dict:
groups_dict = self.group_dict
for n, g in groups_dict.items():
groups[n] = _get_inventory_element(Group, g, n, defaults)
for h in hosts.values():
h.groups = ParentGroups([groups[g] for g in h.groups])
for g in groups.values():
g.groups = ParentGroups([groups[g] for g in g.groups])
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self) -> Inventory:
"""Return nornir Inventory object."""
serialized_defaults = _get_defaults(self.defaults)
serialized_hosts = {}
for host_name, host_data in self.hosts.items():
serialized_hosts[host_name] = _get_inventory_element(
Host, host_data, host_name, serialized_defaults)
serialized_groups = {}
for group_name, group_data in self.groups.items():
serialized_groups[group_name] = _get_inventory_element(
Group, group_data, group_name, serialized_defaults)
for h in serialized_hosts.values():
h.groups = ParentGroups([serialized_groups[g]
for g in h.groups]) # type: ignore
for g in serialized_groups.values():
g.groups = ParentGroups([serialized_groups[g]
for g in g.groups]) # type: ignore
return Inventory(
hosts=serialized_hosts, # type: ignore
groups=serialized_groups, # type: ignore
defaults=serialized_defaults,
)
def load(self) -> Inventory:
"""Standard Nornir 3 load method boilerplate."""
if not self.credentials_params:
self.credentials_params = {}
# Initialize QuerySet
if not self.queryset:
self.queryset = Device.objects.all()
if self.filters:
self.queryset = self.queryset.filter(**self.filters)
if not self.params:
self.params = {}
self.queryset = self.queryset.select_related(
"device_role",
"device_type",
"device_type__manufacturer",
"site",
"platform",
"tenant",
)
# Initialize Hosts and Groups vars
hosts = Hosts()
defaults = Defaults(data=self.defaults)
groups = Groups()
if self.credentials_params:
cred = self.cred_class(params=self.credentials_params)
else:
cred = self.cred_class()
# Create all hosts
for device in self.queryset:
host = self.create_host(device=device,
cred=cred,
params=self.params)
hosts[device.name] = _set_host(data=host["data"],
name=host["name"],
groups=host["groups"],
host=host,
defaults=defaults)
# Initialize all groups if they don't already exist
for group in hosts[device.name].groups:
if group not in groups.keys():
groups[group] = Group(name=group, defaults=defaults)
for _host in hosts.values():
_host.groups = ParentGroups(
[groups[_group] for _group in _host.groups])
for _group in groups.values():
_group.groups = ParentGroups(
[groups[_group] for _group in _group.groups])
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self) -> Inventory:
hosts = Hosts()
groups = Groups()
defaults = Defaults()
hosts_redis = check_redis("nornir_hosts")
groups_redis = check_redis("nornir_groups")
if len(hosts_redis) > 0 and len(groups_redis) > 0:
print(f"Found cached info in Redis and will used that")
serialized_groups = groups_redis
serialized_hosts = hosts_redis
else:
serialized_groups, lnetd_groups = _lnetd_get_groups(self.lnetd_db)
serialized_hosts = _lnetd_get_hosts(self.lnetd_db, lnetd_groups)
for h in serialized_hosts.values():
h.groups = ParentGroups(
[serialized_groups[g] for g in h.groups])
update_redis("nornir_hosts", serialized_hosts)
update_redis("nornir_groups", serialized_groups)
return Inventory(hosts=serialized_hosts,
groups=serialized_groups,
defaults=defaults)
def process_dc_data(group: Group, group_data: Dict[str, Dict[str,
str]]) -> Hosts:
"""
Arguments:
group: Current group we are processing
group_data: the data for the entire group as returned by the backend
"""
# we create a placeholder for the hosts
hosts = Hosts()
# inside each datacenter we had a dictionary where the key was the hostname
# and the data inside had some data about it, we iterate over the hosts
for hostname, host_data in group_data.items():
# for each host we create a Host object mapping it's required parameters
# with the data we got
hosts[hostname] = Host(
name=hostname,
hostname=hostname,
platform=host_data.pop("platform"),
groups=ParentGroups([group
]), # we add the DC group as a parent group
data={
"site": group.name,
**host_data
}, # extra data
)
return hosts
def load(self) -> Inventory:
nr_defaults = _get_defaults(self.defaults)
nr_hosts = {
n: _get_inventory_element(Host, h, n, nr_defaults)
for n, h in self.hosts.items()
}
nr_groups = {
n: _get_inventory_element(Group, g, n, nr_defaults)
for n, g in self.groups.items()
}
for h in nr_hosts.values():
h.groups = ParentGroups([nr_groups[g] for g in h.groups])
for g in nr_groups.values():
g.groups = ParentGroups([groups[g] for g in g.groups])
return Inventory(hosts=nr_hosts, groups=nr_groups, defaults=nr_defaults)
def load(self) -> Inventory:
defaults = _get_defaults(self.defaults_dict)
hosts = Hosts()
hosts_dict = _get_devices_excel(self._excel_file)
for n, h in hosts_dict.items():
hosts[n] = _get_inventory_element(Host, h, n, defaults)
groups = Groups()
for n, g in self.groups_dict.items():
groups[n] = _get_inventory_element(Group, g, n, defaults)
for h in hosts.values():
h.groups = ParentGroups([groups[g] for g in h.groups])
for g in groups.values():
g.groups = ParentGroups([groups[g] for g in g.groups])
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self) -> Inventory:
yml = ruamel.yaml.YAML(typ="safe")
platforms: List[Dict[str, Any]] = []
if self.use_platform_napalm_driver:
platforms = self._get_resources(
url=f"{self.nb_url}/api/dcim/platforms/?limit=0", params={})
nb_devices: List[Dict[str, Any]] = []
nb_devices = self._get_resources(
url=f"{self.nb_url}/api/dcim/devices/?limit=0",
params=self.filter_parameters,
)
if self.include_vms:
nb_devices.extend(
self._get_resources(
url=
f"{self.nb_url}/api/virtualization/virtual-machines/?limit=0",
params=self.filter_parameters,
))
hosts = Hosts()
groups = Groups()
defaults = Defaults()
if self.defaults_file.exists():
with self.defaults_file.open("r") as f:
defaults_dict = yml.load(f) or {}
defaults = _get_defaults(defaults_dict)
else:
defaults = Defaults()
if self.group_file.exists():
with self.group_file.open("r") as f:
groups_dict = yml.load(f) or {}
for n, g in groups_dict.items():
groups[n] = _get_inventory_element(Group, g, n, defaults)
for g in groups.values():
g.groups = ParentGroups([groups[g] for g in g.groups])
for device in nb_devices:
serialized_device: Dict[Any, Any] = {}
serialized_device["data"] = device
if self.flatten_custom_fields:
for cf, value in device["custom_fields"].items():
serialized_device["data"][cf] = value
serialized_device["data"].pop("custom_fields")
hostname = None
if device.get("primary_ip"):
hostname = device.get("primary_ip", {}).get("address",
"").split("/")[0]
else:
if device.get("name") is not None:
hostname = device["name"]
serialized_device["hostname"] = hostname
if isinstance(device["platform"], dict) and self.use_platform_slug:
platform = device["platform"].get("slug")
elif (isinstance(device["platform"], dict)
and self.use_platform_napalm_driver):
platform = [
platform for platform in platforms
if device["platform"]["slug"] == platform["slug"]
][0]["napalm_driver"]
elif isinstance(device["platform"], dict):
platform = device["platform"].get("name")
else:
platform = device["platform"]
serialized_device["platform"] = platform
name = serialized_device["data"].get("name") or str(
serialized_device["data"].get("id"))
hosts[name] = _get_inventory_element(Host, serialized_device, name,
defaults)
groups_extracted = self._extract_device_groups(device)
for group in groups_extracted:
if group not in groups.keys():
groups[group] = _get_inventory_element(
Group, {}, group, defaults)
hosts[name].groups = ParentGroups(
[groups[g] for g in groups_extracted])
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def inventory_from_yaml():
dir_path = os.path.dirname(os.path.realpath(__file__))
yml = ruamel.yaml.YAML(typ="safe")
def get_connection_options(data):
cp = {}
for cn, c in data.items():
cp[cn] = ConnectionOptions(
hostname=c.get("hostname"),
port=c.get("port"),
username=c.get("username"),
password=c.get("password"),
platform=c.get("platform"),
extras=c.get("extras"),
)
return cp
def get_defaults():
defaults_file = f"{dir_path}/inventory_data/defaults.yaml"
with open(defaults_file, "r") as f:
defaults_dict = yml.load(f)
defaults = Defaults(
hostname=defaults_dict.get("hostname"),
port=defaults_dict.get("port"),
username=defaults_dict.get("username"),
password=defaults_dict.get("password"),
platform=defaults_dict.get("platform"),
data=defaults_dict.get("data"),
connection_options=get_connection_options(
defaults_dict.get("connection_options", {})),
)
return defaults
def get_inventory_element(typ, data, name, defaults):
return typ(
name=name,
hostname=data.get("hostname"),
port=data.get("port"),
username=data.get("username"),
password=data.get("password"),
platform=data.get("platform"),
data=data.get("data"),
groups=data.get(
"groups"
), # this is a hack, we will convert it later to the correct type
defaults=defaults,
connection_options=get_connection_options(
data.get("connection_options", {})),
)
host_file = f"{dir_path}/inventory_data/hosts.yaml"
group_file = f"{dir_path}/inventory_data/groups.yaml"
defaults = get_defaults()
hosts = Hosts()
with open(host_file, "r") as f:
hosts_dict = yml.load(f)
for n, h in hosts_dict.items():
hosts[n] = get_inventory_element(Host, h, n, defaults)
groups = Groups()
with open(group_file, "r") as f:
groups_dict = yml.load(f)
for n, g in groups_dict.items():
groups[n] = get_inventory_element(Group, g, n, defaults)
for h in hosts.values():
h.groups = ParentGroups([groups[g] for g in h.groups])
for g in groups.values():
g.groups = ParentGroups([groups[g] for g in g.groups])
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self):
"""Load inventory by fetching devices from netbox."""
if self.filter_parameters:
devices: List[pynetbox.modules.dcim.
Devices] = self.session.dcim.devices.filter(
**self.filter_parameters)
else:
devices: List[pynetbox.modules.dcim.
Devices] = self.session.dcim.devices.all()
# fetch all platforms from Netbox and build mapping: platform: napalm_driver
platforms = self.session.dcim.platforms.all()
platforms_mapping = {
platform.slug: platform.napalm_driver
for platform in platforms if platform.napalm_driver
}
hosts = Hosts()
groups = Groups()
defaults = Defaults()
for dev in devices:
# Netbox allows devices to be unnamed, but the Nornir model does not allow this
# If a device is unnamed we will set the name to the id of the device in netbox
dev_name = dev.name or dev.id
host = NetworkImporterHost(name=dev_name,
connection_options=ConnectionOptions())
# Only add virtual chassis master as inventory element
if dev.virtual_chassis and dev.virtual_chassis.master:
if dev.id != dev.virtual_chassis.master.id:
continue
host.data["virtual_chassis"] = True
else:
host.data["virtual_chassis"] = False
# If supported_platforms is provided
# skip all devices that do not match the list of supported platforms
# TODO need to see if we can filter when doing the query directly
if self.supported_platforms:
if not dev.platform:
continue
if dev.platform.slug not in self.supported_platforms:
continue
# Add value for IP address
if self.settings.use_primary_ip and dev.primary_ip:
host.hostname = dev.primary_ip.address.split("/")[0]
elif self.settings.use_primary_ip and not dev.primary_ip:
host.is_reachable = False
host.not_reachable_reason = "primary ip not defined in Netbox"
elif not self.settings.use_primary_ip and self.settings.fqdn:
host.hostname = f"{dev.name}.{self.settings.fqdn}"
elif not self.settings.use_primary_ip:
host.hostname = dev.name
else:
host.hostname = dev_name
host.site_name = dev.site.slug
host.data["serial"] = dev.serial
host.data["vendor"] = dev.device_type.manufacturer.slug
host.data["asset_tag"] = dev.asset_tag
host.data["custom_fields"] = dev.custom_fields
host.data["site_id"] = dev.site.id
host.data["device_id"] = dev.id
host.data["role"] = dev.device_role.slug
host.data["model"] = dev.device_type.slug
# Attempt to add 'platform' based of value in 'slug'
if dev.platform and dev.platform.slug in platforms_mapping:
host.connection_options = {
"napalm":
ConnectionOptions(
platform=platforms_mapping[dev.platform.slug])
}
if dev.platform:
host.platform = dev.platform.slug
else:
host.platform = None
host.groups = ParentGroups([self.global_group])
if dev.site.slug not in groups.keys():
groups[dev.site.slug] = {}
if dev.device_role.slug not in groups.keys():
groups[dev.device_role.slug] = {}
if host.hostname and host.platform:
host.is_reachable = True
# Assign temporary dict to outer dict
hosts[dev_name] = host
return Inventory(hosts=hosts, groups=groups, defaults=defaults)
def load(self) -> Inventory:
defaults = Defaults(
connection_options={
"napalm": ConnectionOptions(extras={
"optional_args": {
# args to eAPI HttpsEapiConnection for EOS
"enforce_verification": True,
"context": ssl_context
}
})
}
)
insecure_device_states = [
DeviceState.INIT,
DeviceState.DHCP_BOOT,
DeviceState.PRE_CONFIGURED,
DeviceState.DISCOVERED
]
insecure_connection_options = {
"napalm": ConnectionOptions(extras={
"optional_args": {"enforce_verification": False}
})
}
groups = Groups()
for device_type in list(DeviceType.__members__):
group_name = 'T_'+device_type
groups[group_name] = Group(name=group_name, defaults=defaults)
for device_state in list(DeviceState.__members__):
username, password = self._get_credentials(device_state)
group_name = 'S_'+device_state
groups[group_name] = Group(
name=group_name, username=username, password=password, defaults=defaults)
for group_name in get_groups():
groups[group_name] = Group(name=group_name, defaults=defaults)
hosts = Hosts()
with cnaas_nms.db.session.sqla_session() as session:
instance: Device
for instance in session.query(Device):
hostname = self._get_management_ip(instance.management_ip,
instance.dhcp_ip)
port = None
if instance.port and isinstance(instance.port, int):
port = instance.port
host_groups = [
'T_' + instance.device_type.name,
'S_' + instance.state.name
]
for member_group in get_groups(instance.hostname):
host_groups.append(member_group)
if instance.state in insecure_device_states:
host_connection_options = insecure_connection_options
else:
host_connection_options = None
hosts[instance.hostname] = Host(
name=instance.hostname,
hostname=hostname,
platform=instance.platform,
groups=ParentGroups(groups[g] for g in host_groups),
port=port,
data={
'synchronized': instance.synchronized,
'managed': (True if instance.state == DeviceState.MANAGED else False)
},
connection_options=host_connection_options,
defaults=defaults
)
return Inventory(hosts=hosts, groups=groups, defaults=defaults)