def monitor(self, interval):
while True and not self.terminate:
# We get device IDs every time through, so that we can then re-retrieve the device object.
# The reason for this is because other entities may have changed device (e.g. SDWAN heartbeats)
device_ids = get_all_device_ids()
log_console(f"Monitor: Beginning compliance monitoring for {len(device_ids)} devices")
for device_id in device_ids:
if self.terminate:
break
result, device = get_device(device_id=device_id) # re-retrieve device as it may have been changed
if result != "success":
log_console(f"Compliance Monitor: Error retrieving device from DB. id: {device_id}, error: {device}")
continue
if device["availability"]:
device["os_compliance"] = check_os_compliance(device)
device["config_compliance"] = check_config_compliance(device)
device["last_compliance_check"] = str(datetime.now())[:-3]
set_device(device)
for _ in range(0, int(interval / 10)):
sleep(10)
if self.terminate:
break
log_console("...gracefully exiting monitor:compliance")
def device_info():
if request.method == "GET":
device_name = request.args.get("device")
requested_info = request.args.get("info")
live = request.args.get("live")
if not device_name or not requested_info:
return "Must provide device and info", 400
result, info = get_device(device_name=device_name)
if result == "failed":
return info, 406
device = info
if not live:
get_live_info = False
else:
if live.lower() not in {"true", "false"}:
return "Value of 'live', if specified, must be 'true' or 'false'"
else:
get_live_info = bool(live)
status, result_info = get_device_info(device, requested_info,
get_live_info)
if status == "success":
return result_info, 200
else:
return result_info, 406
def monitor(self, interval):
while True and not self.terminate:
device_ids = get_all_device_ids()
log_console(
f"Monitor: Beginning Configuration monitoring for {len(device_ids)} devices"
)
for device_id in device_ids:
if self.terminate:
break
result, device = get_device(
device_id=device_id
) # re-retrieve device as it may have been changed
if result != "success":
log_console(
f"Configuration Monitor: Error retrieving device from DB. id: {device_id}, error: {device}"
)
continue
try:
result, config = get_device_info(device,
"config",
get_live_info=True)
if result != "success":
log_console(
f"!!! Unable to get device info (config) for {device['name']}"
)
continue
except BaseException as e:
log_console(
f"!!! Exception getting device info in configuration monitoring for {device['name']}: {repr(e)}"
)
continue
# If we made it here, we got the configuration, so store it in the DB
record_device_config(device_id, config["config"]["running"])
log_event(
str(datetime.now())[:-3],
"configuration",
device['name'],
"INFO",
f"Stored configuration for: {device['name']}",
)
for _ in range(0, int(interval / 10)):
sleep(10)
if self.terminate:
break
log_console("...gracefully exiting monitor:configuration")
def device_status():
device_name = request.args.get("device")
num_datapoints = request.args.get("datapoints")
if not device_name or not num_datapoints:
return "Must provide deviceid and datapoints", 400
result, info = get_device(device_name=device_name)
if result != "success":
return "Could not find device in DB", 400
device = info
return {
"device_data": get_device_status_data(device_name, num_datapoints),
"device": device,
}
def device_heartbeat():
heartbeat_info = request.get_json()
if not heartbeat_info:
return "Must provide heartbeat information in JSON body", 400
if "serial" not in heartbeat_info:
return "Must provide 'serial' in heartbeat information", 400
if "name" not in heartbeat_info:
return "Must provide 'name' in heartbeat information", 400
result, device = get_device(device_name=heartbeat_info["name"])
if result != "success":
return "Unknown device name in heartbeat information", 400
if heartbeat_info["serial"] != device["serial"]:
return "Serial number in heartbeat information does not match device serial", 400
device["availability"] = True
device["last_heard"] = str(datetime.now())[:-3]
if "vendor" in heartbeat_info:
device["vendor"] = heartbeat_info["vendor"]
if "model" in heartbeat_info:
device["model"] = heartbeat_info["model"]
if "os" in heartbeat_info:
device["os"] = heartbeat_info["os"]
if "version" in heartbeat_info:
device["version"] = heartbeat_info["version"]
if "response_time" in heartbeat_info:
device["response_time"] = heartbeat_info["response_time"]
if "cpu" in heartbeat_info:
device["cpu"] = heartbeat_info["cpu"]
if "memory" in heartbeat_info:
device["memory"] = heartbeat_info["memory"]
if "uptime" in heartbeat_info:
device["uptime"] = heartbeat_info["uptime"]
record_device_status(device)
set_device(device)
log_console(
f"Received heartbeat from {heartbeat_info['name']}, info={heartbeat_info}"
)
return {}, 200
def device_config_diff():
device_name = request.args.get("device")
num_configs = request.args.get("configs", 10)
if not device_name:
return "Must provide device name", 400
result, info = get_device(device_name=device_name)
if result == "failed":
return info, 406
device = info
status, result_info = get_device_config_diff(device, num_configs)
if status == "success":
return result_info, 200
else:
return result_info, 406
def traceroute_register():
registration_info = request.get_json()
if not registration_info:
return "Must provide registration information in JSON body", 400
if "serial" not in registration_info:
return "Must provide 'serial' in registration information", 400
if "name" not in registration_info:
return "Must provide 'name' in registration information", 400
result, device = get_device(device_name=registration_info["name"])
if result != "success":
return "Unknown device name in registration information", 400
if registration_info["serial"] != device["serial"]:
return "Serial number in registration information does not match device serial", 400
log_console(
f"Received registration request from {registration_info['name']}, serial no: {registration_info['serial']}"
)
return {}, 200
def device_register():
registration_info = request.get_json()
if not registration_info:
return "Must provide registration information in JSON body", 400
if "serial" not in registration_info:
return "Must provide 'serial' in registration information", 400
if "name" not in registration_info:
return "Must provide 'name' in registration information", 400
result, device = get_device(device_name=registration_info["name"])
if result != "success":
return "Unknown device name in registration information", 400
if registration_info["serial"] != device["serial"]:
return "Serial number in registration information does not match device serial", 400
log_console(
f"Received registration request from {registration_info['name']}, serial no: {registration_info['serial']}"
)
device["availability"] = True
device["last_heard"] = str(datetime.now())[:-3]
set_device(device)
return {}, 200
def monitor(self, interval):
while True and not self.terminate:
# We get device IDs every time through, so that we can then re-retrieve the device object.
# The reason for this is because other entities may have changed device (e.g. SDWAN heartbeats)
device_ids = get_all_device_ids()
log_console(
f"Monitor: Beginning monitoring for {len(device_ids)} devices")
for device_id in device_ids:
result, device = get_device(
device_id=device_id
) # re-retrieve device as it may have been changed
if result != "success":
log_console(
f"Device Monitor: Error retrieving device from DB. id: {device_id}, error: {device}"
)
continue
if device["transport"] == "HTTP-REST":
if not device["last_heard"]:
continue
last_heard_time = datetime.strptime(
device["last_heard"], "%Y-%m-%d %H:%M:%S.%f")
print(
f"now: {datetime.now()}, last_heard: {last_heard_time}"
)
if (datetime.now() - last_heard_time) > timedelta(
seconds=MAX_NOT_HEARD_SECONDS):
device["availability"] = False
record_device_status(device)
set_device(device)
continue # HTTP-REST devices (e.g. sdwan) communicate to us, we don't poll them
try:
ip_address = socket.gethostbyname(device["hostname"])
except (socket.error, socket.gaierror) as e:
info = f"!!! Caught socket error {repr(e)}, continuing to next device"
log_console(info)
log_event(
str(datetime.now())[:-3], "device", device['name'],
"SEVERE", info)
ip_address = None
if self.terminate:
break
log_console(
f"--- monitor:device get environment {device['name']}")
device_status = get_device_status(device)
device["ip_address"] = ip_address
device["availability"] = device_status["availability"]
device["response_time"] = device_status["response_time"]
device["cpu"] = device_status["cpu"]
device["memory"] = device_status["memory"]
if device_status["last_heard"]:
device["last_heard"] = device_status["last_heard"]
record_device_status(device)
set_device(device)
for _ in range(0, int(interval / 10)):
sleep(10)
if self.terminate:
break
log_console("...gracefully exiting monitor:device")