def single_request(self, this_request):
"""
Actually perform the request and retry x times if request times out.
Program will exit if all retries failed!
Parameters
----------
this_request: requests.session.prepare_request
object of the prepared request
Returns
-------
requests.Response: response for this request
"""
response = None
if log.level == DEBUG3:
pprint.pprint(vars(this_request))
for _ in range(self.max_retry_attempts):
log_message = f"Sending {this_request.method} to '{this_request.url}'"
if this_request.body is not None:
log_message += f" with data '{this_request.body}'."
log.debug2(log_message)
try:
response = self.session.send(this_request,
timeout=self.timeout,
verify=self.validate_tls_certs)
except (ConnectionError, requests.exceptions.ConnectionError,
requests.exceptions.ReadTimeout):
log.warning(f"Request failed, trying again: {log_message}")
continue
else:
break
else:
do_error_exit(
f"Giving up after {self.max_retry_attempts} retries.")
log.debug2("Received HTTP Status %s.", response.status_code)
# print debugging information
if log.level == DEBUG3:
log.debug("Response Body:")
try:
pprint.pprint(response.json())
except json.decoder.JSONDecodeError as e:
log.error(e)
return response
def resolve_relations(self):
o_id = self.data.get("assigned_object_id")
o_type = self.data.get("assigned_object_type")
# this needs special treatment as the object type depends on a second model key
if o_type is not None and o_type not in self.data_model.get(
"assigned_object_type"):
log.error(
f"Attribute 'assigned_object_type' for '{self.get_display_name()}' invalid: {o_type}"
)
do_error_exit(
f"Error while resolving relations for {self.get_display_name()}"
)
if isinstance(o_id, int):
self.data["assigned_object_id"] = self.inventory.get_by_id(
self.data_model_relation.get(o_type), nb_id=o_id)
super().resolve_relations()
def open_config_file(config_file):
"""
Open config file with a ConfigParser and return handler. Bail out of opening or parsing fails
Parameters
----------
config_file: str
absolute path of config file to open
Returns
-------
ConfigParser: handler with supplied config file
"""
if config_file is None or config_file == "":
do_error_exit("ERROR: Config file not defined.")
# setup config parser and read config
config_handler = configparser.ConfigParser(strict=True, allow_no_value=True, empty_lines_in_values=False)
# noinspection PyBroadException
try:
config_handler.read_file(open(config_file))
except configparser.Error as e:
do_error_exit(f"ERROR: Problem while config file parsing: {e}")
# noinspection PyBroadException
except Exception:
do_error_exit(f"ERROR: Unable to open file '{config_file}'")
return config_handler
def get_config_file(config_file):
"""
get absolute path to provided config file string
Parameters
----------
config_file: str
config file path
Returns
-------
str: absolute path to config file
"""
if config_file is None or config_file == "":
do_error_exit("ERROR: Config file not defined.")
base_dir = os.sep.join(__file__.split(os.sep)[0:-3])
if config_file[0] != os.sep:
config_file = f"{base_dir}{os.sep}{config_file}"
return os.path.realpath(config_file)
def __init__(self, settings=None, inventory=None):
self.settings = settings
self.inventory = inventory
self.parse_config_settings(settings)
# flood the console
if log.level == DEBUG3:
log.warning(
"Log level is set to DEBUG3, Request logs will only be printed to console"
)
HTTPConnection.debuglevel = 1
proto = "https"
if bool(self.disable_tls) is True:
proto = "http"
# disable TLS insecure warnings if user explicitly switched off validation
if bool(self.validate_tls_certs) is False:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
port = ""
if self.port is not None:
port = f":{self.port}"
self.url = f"{proto}://{self.host_fqdn}{port}/api/"
self.session = self.create_session()
# check for minimum version
api_version = self.get_api_version()
if version.parse(api_version) < version.parse(
self.minimum_api_version):
do_error_exit(f"Netbox API version '{api_version}' not supported. "
f"Minimum API version: {self.minimum_api_version}")
self.setup_caching()
def get_api_version(self):
"""
Perform a basic GET request to extract NetBox API version from header
Returns
-------
str: NetBox API version
"""
response = None
try:
response = self.session.get(self.url,
timeout=self.timeout,
verify=self.validate_tls_certs)
except Exception as e:
do_error_exit(str(e))
result = str(response.headers["API-Version"])
log.info(f"Successfully connected to NetBox '{self.host_fqdn}'")
log.debug(f"Detected NetBox API version: {result}")
return result
def setup_logging(log_level=None, log_file=None):
"""
Set up logging for the whole program and return a log handler
Parameters
----------
log_level: str
valid log level to set logging to
log_file: str
name of the log file to log to
Returns
-------
log handler to use for logging
"""
log_file_max_size_in_mb = 10
log_file_max_rotation = 5
log_format = '%(asctime)s - %(levelname)s: %(message)s'
if log_level is None or log_level == "":
do_error_exit("ERROR: log level undefined or empty. Check config please.")
# check set log level against self defined log level array
if not log_level.upper() in valid_log_levels:
do_error_exit(f"ERROR: Invalid log level: {log_level}")
# check the provided log level
if log_level == "DEBUG2":
numeric_log_level = DEBUG2
elif log_level == "DEBUG3":
numeric_log_level = DEBUG3
logging.basicConfig(level=logging.DEBUG, format=log_format)
else:
numeric_log_level = getattr(logging, log_level.upper(), None)
log_format = logging.Formatter(log_format)
# create logger instance
logger = get_logger()
logger.setLevel(numeric_log_level)
# setup stream handler
# in DEBUG3 the root logger gets redefined, that would print every log message twice
if log_level != "DEBUG3":
log_stream = logging.StreamHandler()
log_stream.setFormatter(log_format)
logger.addHandler(log_stream)
# setup log file handler
if log_file is not None:
# base directory is three levels up
base_dir = os.sep.join(__file__.split(os.sep)[0:-3])
if log_file[0] != os.sep:
log_file = f"{base_dir}{os.sep}{log_file}"
log_file_handler = None
try:
log_file_handler = RotatingFileHandler(
filename=log_file,
maxBytes=log_file_max_size_in_mb * 1024 * 1024, # Bytes to Megabytes
backupCount=log_file_max_rotation
)
except Exception as e:
do_error_exit(f"ERROR: Problems setting up log file: {e}")
log_file_handler.setFormatter(log_format)
logger.addHandler(log_file_handler)
return logger
def query_current_data(self, netbox_objects_to_query=None):
"""
Request all current NetBox objects. Use caching whenever possible.
Objects must provide "last_updated" attribute to support caching for this object type.
Otherwise it's not possible to query only changed objects since last run. If attribute is
not present all objects will be requested (looking at you *Interfaces)
Parameters
----------
netbox_objects_to_query: list of NetBoxObject sub classes
NetBox items to query
"""
if netbox_objects_to_query is None:
raise AttributeError(
f"Attribute netbox_objects_to_query is: '{netbox_objects_to_query}'"
)
# query all dependencies
for nb_object_class in netbox_objects_to_query:
if nb_object_class not in NetBoxObject.__subclasses__():
raise AttributeError(
f"Class '{nb_object_class.__name__}' must be a "
f"subclass of '{NetBoxObject.__name__}'")
# if objects are multiple times requested but already retrieved
if nb_object_class in self.resolved_dependencies:
continue
# initialize cache variables
cached_nb_data = list()
cache_file = f"{self.cache_directory}{os.sep}{nb_object_class.__name__}.cache"
cache_this_class = False
latest_update = None
# check if cache file is accessible
if self.use_caching is True:
cache_this_class = True
if os.path.exists(cache_file) and not os.access(
cache_file, os.R_OK):
log.warning(
f"Got no permission to read existing cache file: {cache_file}"
)
cache_this_class = False
if os.path.exists(cache_file) and not os.access(
cache_file, os.W_OK):
log.warning(
f"Got no permission to write to existing cache file: {cache_file}"
)
cache_this_class = False
# read data from cache file
if cache_this_class is True:
# noinspection PyBroadException
try:
cached_nb_data = pickle.load(open(cache_file, "rb"))
except Exception:
pass
if cached_nb_data is None:
cached_nb_data = list()
# get date of latest update in cache file
if len(cached_nb_data) > 0:
latest_update_list = \
[x.get("last_updated") for x in cached_nb_data if x.get("last_updated") is not None]
if len(latest_update_list) > 0:
latest_update = sorted(latest_update_list)[-1]
log.debug(
f"Successfully read cached data with {len(cached_nb_data)} '{nb_object_class.name}%s'"
f", last updated '{latest_update}'" %
plural(len(cached_nb_data)))
elif self.testing_cache is False:
cache_this_class = False
if self.testing_cache is True and len(cached_nb_data) > 0:
for object_data in cached_nb_data:
self.inventory.add_object(nb_object_class,
data=object_data,
read_from_netbox=True)
# mark this object class as retrieved
self.resolved_dependencies.add(nb_object_class)
continue
full_nb_data = None
brief_nb_data = None
updated_nb_data = None
# no cache data found
if latest_update is None:
# get all objects of this class
log.debug(
f"Requesting all {nb_object_class.name}s from NetBox")
full_nb_data = self.request(nb_object_class)
if full_nb_data.get("results") is None:
log.error(
f"Result data from NetBox for object {nb_object_class.__name__} missing!"
)
do_error_exit("Reading data from NetBox failed.")
else:
# request a brief list of existing objects
log.debug(
f"Requesting a brief list of {nb_object_class.name}s from NetBox"
)
brief_nb_data = self.request(nb_object_class,
params={
"brief": 1,
"limit": 500
})
log.debug("NetBox returned %d results." %
len(brief_nb_data.get("results", list())))
log.debug(
f"Requesting the last updates since {latest_update} of {nb_object_class.name}s from NetBox"
)
updated_nb_data = self.request(
nb_object_class,
params={"last_updated__gte": latest_update})
log.debug("NetBox returned %d results." %
len(updated_nb_data.get("results", list())))
if brief_nb_data.get("results") is None or updated_nb_data.get(
"results") is None:
log.error(
f"Result data from NetBox for object {nb_object_class.__name__} missing!"
)
do_error_exit("Reading data from NetBox failed.")
# read a full set from NetBox
nb_objects = list()
if full_nb_data is not None:
nb_objects = full_nb_data.get("results")
elif self.testing_cache is True:
nb_objects = cached_nb_data
# read the delta from NetBox and
else:
currently_existing_ids = [
x.get("id") for x in brief_nb_data.get("results")
]
changed_ids = [
x.get("id") for x in updated_nb_data.get("results")
]
for this_object in cached_nb_data:
if this_object.get(
"id"
) in currently_existing_ids and this_object.get(
"id") not in changed_ids:
nb_objects.append(this_object)
nb_objects.extend(updated_nb_data.get("results"))
if self.use_caching is True:
try:
pickle.dump(nb_objects, open(cache_file, "wb"))
if cache_this_class is True:
log.debug("Successfully cached %d objects." %
(len(nb_objects)))
except Exception as e:
log.warning(
f"Failed to write NetBox data to cache file: {e}")
log.debug(f"Processing %s returned {nb_object_class.name}%s" %
(len(nb_objects), plural(len(nb_objects))))
for object_data in nb_objects:
self.inventory.add_object(nb_object_class,
data=object_data,
read_from_netbox=True)
# mark this object class as retrieved
self.resolved_dependencies.add(nb_object_class)
def request(self,
object_class,
req_type="GET",
data=None,
params=None,
nb_id=None):
"""
Perform a NetBox request for a certain object.
Parameters
----------
object_class: NetBoxObject sub class
class definition of the desired NetBox object
req_type: str
GET, PATCH, PUT, DELETE
data: dict
data which shall be send to NetBox
params: dict
dict of URL params which should be passed to NetBox
nb_id: int
ID of the NetBox object which will be appended to the requested NetBox URL
Returns
-------
(dict, bool, None): of returned NetBox data. If object was requested to be deleted and it was
successful then True will be returned. None if request failed or was empty
"""
result = None
request_url = f"{self.url}{object_class.api_path}/"
# append NetBox ID
if nb_id is not None:
request_url += f"{nb_id}/"
if params is not None and not isinstance(params, dict):
log.debug(
f"Params passed to NetBox request need to be a dict, got: {params}"
)
params = dict()
if req_type == "GET":
if params is None:
params = dict()
if "limit" not in params.keys():
params["limit"] = self.default_netbox_result_limit
# always exclude config context
params["exclude"] = "config_context"
# prepare request
this_request = self.session.prepare_request(
requests.Request(req_type, request_url, params=params, json=data))
# issue request
response = self.single_request(this_request)
try:
result = response.json()
except json.decoder.JSONDecodeError:
pass
if response.status_code == 200:
# retrieve paginated results
if this_request.method == "GET" and result is not None:
while response.json().get("next") is not None:
this_request.url = response.json().get("next")
log.debug2(
"NetBox results are paginated. Getting next page")
response = self.single_request(this_request)
result["results"].extend(response.json().get("results"))
elif response.status_code in [201, 204]:
action = "created" if response.status_code == 201 else "deleted"
if req_type == "DELETE":
object_name = self.inventory.get_by_id(object_class, nb_id)
if object_name is not None:
object_name = object_name.get_display_name()
else:
object_name = result.get(object_class.primary_key)
log.info(
f"NetBox successfully {action} {object_class.name} object '{object_name}'."
)
if response.status_code == 204:
result = True
# token issues
elif response.status_code == 403:
do_error_exit("NetBox returned: %s: %s" %
(response.reason, grab(result, "detail")))
# we screw up something else
elif 400 <= response.status_code < 500:
log.error(
f"NetBox returned: {this_request.method} {this_request.path_url} {response.reason}"
)
log.error(f"NetBox returned body: {result}")
result = None
elif response.status_code >= 500:
do_error_exit(
f"NetBox returned: {response.status_code} {response.reason}")
return result
