def update(self, data=None, read_from_netbox=False, source=None):
# don't change the name of the VLAN if it already exists
if read_from_netbox is False and grab(self, "data.name") is not None:
data["name"] = grab(self, "data.name")
super().update(data=data,
read_from_netbox=read_from_netbox,
source=source)
def update_tags(self, tags, remove=False):
"""
Update list of object tags
Parameters
----------
tags: (str, list, dict, NBTag)
tags to parse and add/remove to/from current list of object tags
remove: bool
True if tags shall be removed, otherwise they will be added
Returns
-------
None
"""
if tags is None or NBTagList not in self.data_model.values():
return
action = "Adding" if remove is False else "Removing"
log.debug2(f"{action} Tags: {tags}")
current_tags = grab(self, "data.tags", fallback=NBTagList())
new_tags = self.compile_tags(tags, remove=remove)
if str(current_tags.get_display_name()) != str(
new_tags.get_display_name()):
self.data["tags"] = new_tags
self.updated_items.append("tags")
log.info(
f"{self.name.capitalize()} '{self.get_display_name()}' attribute 'tags' changed from "
f"'{current_tags.get_display_name()}' to '{new_tags.get_display_name()}'"
)
def main():
start_time = datetime.now()
# parse command line
args = parse_command_line(
self_description=self_description,
version=__version__,
version_date=__version_date__,
default_config_file_path=default_config_file_path)
# get config file path
config_file = get_config_file(args.config_file)
# get config handler
config_handler = open_config_file(config_file)
# get logging configuration
# set log level
log_level = default_log_level
# config overwrites default
log_level = config_handler.get("common", "log_level", fallback=log_level)
# cli option overwrites config file
log_level = grab(args, "log_level", fallback=log_level)
log_file = None
if bool(config_handler.getboolean("common", "log_to_file",
fallback=False)) is True:
log_file = config_handler.get("common", "log_file", fallback=None)
# setup logging
log = setup_logging(log_level, log_file)
# now we are ready to go
log.info("Starting " + __description__)
log.debug(f"Using config file: {config_file}")
# initialize an empty inventory which will be used to hold and reference all objects
inventory = NetBoxInventory()
# get config for NetBox handler
netbox_settings = get_config(config_handler,
section="netbox",
valid_settings=NetBoxHandler.settings)
# establish NetBox connection
nb_handler = NetBoxHandler(settings=netbox_settings, inventory=inventory)
# if purge was selected we go ahead and remove all items which were managed by this tools
if args.purge is True:
if args.dry_run is True:
do_error_exit("Purge not available with option 'dry_run'")
nb_handler.just_delete_all_the_things()
# that's it, we are done here
exit(0)
# instantiate source handlers and get attributes
log.info("Initializing sources")
sources = instantiate_sources(config_handler, inventory)
# all sources are unavailable
if len(sources) == 0:
log.error("No working sources found. Exit.")
exit(1)
# collect all dependent object classes
log.info(
"Querying necessary objects from Netbox. This might take a while.")
for source in sources:
nb_handler.query_current_data(source.dependent_netbox_objects)
log.info("Finished querying necessary objects from Netbox")
# resolve object relations within the initial inventory
inventory.resolve_relations()
# initialize basic data needed for syncing
nb_handler.initialize_basic_data()
# loop over sources and patch netbox data
for source in sources:
source.apply()
# add/remove tags to/from all inventory items
inventory.tag_all_the_things(nb_handler)
# update all IP addresses
inventory.query_ptr_records_for_all_ips()
if args.dry_run is True:
log.info("This is a dry run and we stop here. Running time: %s" %
get_relative_time(datetime.now() - start_time))
exit(0)
# update data in NetBox
nb_handler.update_instance()
# prune orphaned objects from NetBox
nb_handler.prune_data()
# finish
log.info("Completed NetBox Sync in %s" %
get_relative_time(datetime.now() - start_time))
def prune_data(self):
"""
Prune objects in NetBox if they are no longer present in any source.
First they will be marked as Orphaned and after X days they will be
deleted from NetBox.
"""
if self.prune_enabled is False:
log.debug("Pruning disabled. Skipping")
return
log.info("Pruning orphaned data in NetBox")
# update all items in NetBox accordingly
today = datetime.now()
for nb_object_sub_class in reversed(NetBoxObject.__subclasses__()):
if getattr(nb_object_sub_class, "prune", False) is False:
continue
for this_object in self.inventory.get_all_items(
nb_object_sub_class):
if this_object.source is not None:
continue
if self.orphaned_tag not in this_object.get_tags():
continue
date_last_update = grab(this_object, "data.last_updated")
if date_last_update is None:
continue
# already deleted
if getattr(this_object, "deleted", False) is True:
continue
# only need the date including seconds
date_last_update = date_last_update[0:19]
log.debug2(
f"Object '{this_object.name}' '{this_object.get_display_name()}' is Orphaned. "
f"Last time changed: {date_last_update}")
# check prune delay.
# noinspection PyBroadException
try:
last_updated = datetime.strptime(date_last_update,
"%Y-%m-%dT%H:%M:%S")
except Exception:
continue
days_since_last_update = (today - last_updated).days
# it seems we need to delete this object
if last_updated is not None and days_since_last_update >= self.prune_delay_in_days:
log.info(
f"{nb_object_sub_class.name.capitalize()} '{this_object.get_display_name()}' is orphaned "
f"for {days_since_last_update} days and will be deleted."
)
# delete device/VM interfaces first. interfaces have no last_updated attribute
if isinstance(this_object, (NBVM, NBDevice)):
log.info(
f"Before the '{this_object.name}' can be deleted, all interfaces must be deleted."
)
for object_interface in self.inventory.get_all_interfaces(
this_object):
# already deleted
if getattr(object_interface, "deleted",
False) is True:
continue
log.info(
f"Deleting interface '{object_interface.get_display_name()}'"
)
ret = self.request(object_interface.__class__,
req_type="DELETE",
nb_id=object_interface.nb_id)
if ret is True:
object_interface.deleted = True
ret = self.request(nb_object_sub_class,
req_type="DELETE",
nb_id=this_object.nb_id)
if ret is True:
this_object.deleted = True
return
def update_object(self, nb_object_sub_class, unset=False, last_run=False):
"""
Iterate over all objects of a certain NetBoxObject sub class and add/update them.
But first update objects which this object class depends on.
If some dependencies are unresolvable then these will be removed from the request
and re added later to the object to try update object in a third run.
Parameters
----------
nb_object_sub_class: NetBoxObject sub class
NetBox objects to update
unset: bool
True if only unset items should be deleted
last_run: bool
True if this will be the last update run. Needed to assign primary_ip4/6 properly
"""
for this_object in self.inventory.get_all_items(nb_object_sub_class):
# resolve dependencies
for dependency in this_object.get_dependencies():
if dependency not in self.resolved_dependencies:
log.debug2("Resolving dependency: %s" % dependency.name)
self.update_object(dependency)
# unset data if requested
if unset is True:
if len(this_object.unset_items) == 0:
continue
unset_data = dict()
for unset_item in this_object.unset_items:
key_data_type = grab(this_object,
f"data_model.{unset_item}")
if key_data_type in NBObjectList.__subclasses__():
unset_data[unset_item] = []
else:
unset_data[unset_item] = None
log.info("Updating NetBox '%s' object '%s' with data: %s" %
(this_object.name, this_object.get_display_name(),
unset_data))
returned_object_data = self.request(nb_object_sub_class,
req_type="PATCH",
data=unset_data,
nb_id=this_object.nb_id)
if returned_object_data is not None:
this_object.update(data=returned_object_data,
read_from_netbox=True)
this_object.resolve_relations()
else:
log.error(
f"Request Failed for {nb_object_sub_class.name}. Used data: {unset_data}"
)
continue
data_to_patch = dict()
unresolved_dependency_data = dict()
for key, value in this_object.data.items():
if key in this_object.updated_items:
if isinstance(value, (NetBoxObject, NBObjectList)):
# resolve dependency issues in last run
# primary IP always set in last run
if value.get_nb_reference() is None or \
(key.startswith("primary_ip") and last_run is False):
unresolved_dependency_data[key] = value
else:
data_to_patch[key] = value.get_nb_reference()
else:
data_to_patch[key] = value
issued_request = False
returned_object_data = None
if len(data_to_patch.keys()) > 0:
# default is a new object
nb_id = None
req_type = "POST"
action = "Creating new"
# if its not a new object then update it
if this_object.is_new is False:
nb_id = this_object.nb_id
req_type = "PATCH"
action = "Updating"
log.info("%s NetBox '%s' object '%s' with data: %s" %
(action, this_object.name,
this_object.get_display_name(), data_to_patch))
returned_object_data = self.request(nb_object_sub_class,
req_type=req_type,
data=data_to_patch,
nb_id=nb_id)
issued_request = True
if returned_object_data is not None:
this_object.update(data=returned_object_data,
read_from_netbox=True)
elif issued_request is True:
log.error(
f"Request Failed for {nb_object_sub_class.name}. Used data: {data_to_patch}"
)
# add unresolved dependencies back to object
if len(unresolved_dependency_data.keys()) > 0:
log.debug2(
"Adding unresolved dependencies back to object: %s" %
list(unresolved_dependency_data.keys()))
this_object.update(data=unresolved_dependency_data)
this_object.resolve_relations()
# add class to resolved dependencies
self.resolved_dependencies.add(nb_object_sub_class)
def request(self,
object_class,
req_type="GET",
data=None,
params=None,
nb_id=None):
"""
Perform a NetBox request for a certain object.
Parameters
----------
object_class: NetBoxObject sub class
class definition of the desired NetBox object
req_type: str
GET, PATCH, PUT, DELETE
data: dict
data which shall be send to NetBox
params: dict
dict of URL params which should be passed to NetBox
nb_id: int
ID of the NetBox object which will be appended to the requested NetBox URL
Returns
-------
(dict, bool, None): of returned NetBox data. If object was requested to be deleted and it was
successful then True will be returned. None if request failed or was empty
"""
result = None
request_url = f"{self.url}{object_class.api_path}/"
# append NetBox ID
if nb_id is not None:
request_url += f"{nb_id}/"
if params is not None and not isinstance(params, dict):
log.debug(
f"Params passed to NetBox request need to be a dict, got: {params}"
)
params = dict()
if req_type == "GET":
if params is None:
params = dict()
if "limit" not in params.keys():
params["limit"] = self.default_netbox_result_limit
# always exclude config context
params["exclude"] = "config_context"
# prepare request
this_request = self.session.prepare_request(
requests.Request(req_type, request_url, params=params, json=data))
# issue request
response = self.single_request(this_request)
try:
result = response.json()
except json.decoder.JSONDecodeError:
pass
if response.status_code == 200:
# retrieve paginated results
if this_request.method == "GET" and result is not None:
while response.json().get("next") is not None:
this_request.url = response.json().get("next")
log.debug2(
"NetBox results are paginated. Getting next page")
response = self.single_request(this_request)
result["results"].extend(response.json().get("results"))
elif response.status_code in [201, 204]:
action = "created" if response.status_code == 201 else "deleted"
if req_type == "DELETE":
object_name = self.inventory.get_by_id(object_class, nb_id)
if object_name is not None:
object_name = object_name.get_display_name()
else:
object_name = result.get(object_class.primary_key)
log.info(
f"NetBox successfully {action} {object_class.name} object '{object_name}'."
)
if response.status_code == 204:
result = True
# token issues
elif response.status_code == 403:
do_error_exit("NetBox returned: %s: %s" %
(response.reason, grab(result, "detail")))
# we screw up something else
elif 400 <= response.status_code < 500:
log.error(
f"NetBox returned: {this_request.method} {this_request.path_url} {response.reason}"
)
log.error(f"NetBox returned body: {result}")
result = None
elif response.status_code >= 500:
do_error_exit(
f"NetBox returned: {response.status_code} {response.reason}")
return result
def compile_tags(self, tags, remove=False):
"""
Parameters
----------
tags: (str, list, dict, NBTag)
tags to parse and add/remove to/from current list of object tags
remove: bool
True if tags shall be removed, otherwise they will be added
Returns
-------
NBTagList: with added/removed tags
"""
if tags is None or NBTagList not in self.data_model.values():
return
# list of parsed tag strings
sanitized_tag_strings = list()
log.debug2(f"Compiling TAG list")
new_tag_list = NBTagList()
def extract_tags(this_tags):
if isinstance(this_tags, NBTag):
sanitized_tag_strings.append(this_tags.get_display_name())
elif isinstance(this_tags, str):
sanitized_tag_strings.append(this_tags)
elif isinstance(this_tags,
dict) and this_tags.get("name") is not None:
sanitized_tag_strings.append(this_tags.get("name"))
if isinstance(tags, list):
for tag in tags:
extract_tags(tag)
else:
extract_tags(tags)
# current list of tag strings
current_tag_strings = self.get_tags()
new_tags = list()
removed_tags = list()
for tag_name in sanitized_tag_strings:
# add tag
if tag_name not in current_tag_strings and remove is False:
tag = self.inventory.add_update_object(NBTag,
data={"name": tag_name})
new_tags.append(tag)
if tag_name in current_tag_strings and remove is True:
tag = self.inventory.get_by_data(NBTag,
data={"name": tag_name})
removed_tags.append(tag)
current_tags = grab(self, "data.tags", fallback=NBTagList())
if len(new_tags) > 0:
for tag in new_tags + current_tags:
new_tag_list.append(tag)
elif len(removed_tags) > 0:
for tag in current_tags:
if tag not in removed_tags:
new_tag_list.append(tag)
else:
new_tag_list = current_tags
return new_tag_list