def cli_process(device_info: Dict[str, str], timestamp: int,
upgrade: bool) -> List[str]:
""" Command line process, ready to run as separate thread or process """
# Setup logger to show process name
netcat.bind_logger(device_info["device_name"].upper())
# Time process execution
start_time = time.time()
# Log initial status
if netcat.SINGLE_PROCESS_MODE:
netcat.LOGGER.opt(ansi=True).info(
"<green>Executing CLI operations as part of main process </green>")
else:
netcat.LOGGER.opt(ansi=True).info(
"<green>Executing CLI operations as child process</green>")
upgrade_software(device_info, REQUESTED_SOFTWARE_VERSION, upgrade)
# Time process execution
end_time = time.time()
# Log process end status and execution time
netcat.LOGGER.opt(ansi=True).info(
f"<green>CLI process ended normaly, execution time: {end_time - start_time:.2f}s</green>"
)
# Return device name to indicate successfuly executed operation for given device
return [device_info["device_name"]]
def get_failed_device_data_list(
device_list: List[Tuple[str, str]]) -> List[Dict[str, Any]]:
""" Prepare list of device data documents to be used to create failed device data list """
# Time process execution
start_time = time.monotonic()
# Setup logger to show process name
if os.getpid() != netcat.MAIN_PROCESS_PID:
netcat.bind_logger("SUB_PROC")
# Pull list of latest device_data documents for devices in device_list
device_data_list = db.get_device_data_list__b([_[0] for _ in device_list],
command_list=None)
# Create mock entries for devices that exist in command snapshot status but were never pooled successfuly
for device_name in set([_[0] for _ in device_list]) - set(
_.get("device_name") for _ in device_data_list):
device_data_list.append({
"device_name":
device_name,
"device_type":
next(iter([_[1] for _ in device_list if _[0] == device_name]),
None)
})
# Time process execution
end_time = time.monotonic()
netcat.LOGGER.debug(
f"Created inaccessible device list for {len(device_list)} devices in {end_time - start_time:.2f}s"
)
return device_data_list
def cli_process(device_info: Dict[str, str], timestamp: int, force_backup: bool = False, test_run: bool = False) -> List[str]:
""" Command line process, ready to run as separate thread or process """
# Setup logger to show process name
netcat.bind_logger(device_info["device_name"].upper())
# Time process execution
start_time = time.time()
# Log initial status
if netcat.SINGLE_PROCESS_MODE:
netcat.LOGGER.opt(ansi=True).info("<green>Executing CLI operations as part of main process </green>")
else:
netcat.LOGGER.opt(ansi=True).info("<green>Executing CLI operations as child process</green>")
# Access device and retrieve its current configuration in all formats, update existing device_dat structure
device_data = get_device_data(device_info)
if not test_run:
# Get list of previous configuration backups stored localy and compare latest of them with current config set
config_change = detect_config_change(device_data)
# Save device info to databae
save_device_data(device_data, timestamp, config_change, force_backup)
# Time process execution
end_time = time.time()
# Log process end status and execution time
netcat.LOGGER.opt(ansi=True).info(f"<green>CLI process ended normaly, execution time: {end_time - start_time:.2f}s</green>")
# Return device name to indicate successfuly executed operation for given device
return [device_info["device_name"]]
def find_mac_address_in_snapshot(timestamp: int, mac_address: str, physical_ports_only: bool) -> List[Dict[str, str]]:
""" Execute 'find_ip_address_in_device_data()' function on all devices info files with given timestamp """
# Time process execution
start_time = time.monotonic()
netcat.bind_logger("SUB_PROC")
device_data_list = db.get_device_data_list__a(timestamp, ["cisco_switch", "cisco_nexus"], command_list=["show mac address-table"])
findings = [_ for __ in device_data_list for _ in find_mac_address_in_device_data(__, mac_address, physical_ports_only)]
# Time process execution
end_time = time.monotonic()
netcat.LOGGER.debug(f"Search for '{mac_address}' performed on {len(device_data_list)} devices in '{timestamp}' snapshot in {end_time - start_time:.2f}s")
return findings
def find_ip_address_in_snapshot(timestamp: int, ip_address: str, use_arp: bool,
use_dhcp: bool,
use_dsnp: bool) -> List[Dict[str, str]]:
""" Execute 'find_ip_address_in_device_data()' function on all devices info files with given timestamp """
# Time process execution
start_time = time.monotonic()
netcat.bind_logger("SUB_PROC")
device_type_list: Set[str] = set()
command_list: Set[str] = set()
if use_arp:
device_type_list |= {"cisco_router", "paloalto"}
command_list |= {"show ip arp", "show arp all"}
if use_dhcp:
device_type_list |= {"cisco_router", "paloalto"}
command_list |= {
"show ip dhcp binding", "show dhcp server lease interface all"
}
if use_dsnp:
device_type_list |= {"cisco_switch"}
command_list |= {"show ip dhcp snooping binding"}
device_data_list = db.get_device_data_list__a(timestamp,
list(device_type_list),
list(command_list))
findings = [
_ for __ in device_data_list for _ in find_ip_address_in_device_data(
__, ip_address, use_arp, use_dhcp, use_dsnp)
]
# Time process execution
end_time = time.monotonic()
netcat.LOGGER.debug(
f"Search for '{ip_address}' performed on {len(device_data_list)} devices in '{timestamp}' snapshot in {end_time - start_time:.2f}s"
)
return findings
async def _(hostname):
try:
await resolver.query(hostname, "A")
except aiodns.error.DNSError as exception:
if exception.args[0] == 4:
netcat.bind_logger(dns_info["description"])
netcat.LOGGER.info(f"Not able to resolve '{hostname}'")
return "FAIL [R]"
if exception.args[0] == 12:
netcat.bind_logger(dns_info["description"])
netcat.LOGGER.info(
f"Not able to connect to server trying to resolve '{hostname}'"
)
return "FAIL [C]"
netcat.LOGGER.info(
f"Unknown error trying to resolve '{hostname}' - '{exception.args[1]}'"
)
return "FAIL [U]"
netcat.bind_logger(dns_info["description"])
netcat.LOGGER.info(f"Successfully resolved '{hostname}'")
return "OK [CR]"
def cli_process(device_info: Dict[str, str], snippet: str, site_id_check: bool, inet_gw_check: bool, no_commit: bool) -> List[Dict[str, str]]:
""" Command line process, ready to run as separate thread or process """
# Setup logger to show process name
netcat.bind_logger(device_info["device_name"].upper())
# Time process execution
start_time = time.time()
# Log initial status
netcat.LOGGER.opt(ansi=True).info("<green>Starting CLI process</green>")
# Deploy configuration snippet
deploy_config_snippet(device_info, snippet, site_id_check, inet_gw_check, no_commit)
# Time process execution
end_time = time.time()
# Log process end status and execution time
netcat.LOGGER.opt(ansi=True).info(f"<green>CLI process ended normaly, execution time: {end_time - start_time:.2f}s</green>")
return [device_info["device_name"]]
async def dns_check(dns_info: Dict[str, str]) -> None:
""" Perform DNS check """
dns_data = {
"description": dns_info["description"],
"ip_address": dns_info["ip_address"],
"results": {}
}
async def _(hostname):
try:
await resolver.query(hostname, "A")
except aiodns.error.DNSError as exception:
if exception.args[0] == 4:
netcat.bind_logger(dns_info["description"])
netcat.LOGGER.info(f"Not able to resolve '{hostname}'")
return "FAIL [R]"
if exception.args[0] == 12:
netcat.bind_logger(dns_info["description"])
netcat.LOGGER.info(
f"Not able to connect to server trying to resolve '{hostname}'"
)
return "FAIL [C]"
netcat.LOGGER.info(
f"Unknown error trying to resolve '{hostname}' - '{exception.args[1]}'"
)
return "FAIL [U]"
netcat.bind_logger(dns_info["description"])
netcat.LOGGER.info(f"Successfully resolved '{hostname}'")
return "OK [CR]"
resolver = aiodns.DNSResolver(timeout=0)
resolver.nameservers = [dns_info["ip_address"]]
netcat.bind_logger(dns_info["description"])
netcat.LOGGER.info(f"Querrying server for '{HOSTNAME_EXTERNAL}'")
dns_data["results"]["external"] = await _(HOSTNAME_EXTERNAL)
netcat.bind_logger(dns_info["description"])
netcat.LOGGER.info(f"Querrying server for '{HOSTNAME_INTERNAL}'")
dns_data["results"]["internal"] = await _(HOSTNAME_INTERNAL)
netcat.bind_logger("MAIN_PROG")
return dns_data
async def main() -> int:
""" Main program """
timestamp = int((datetime.datetime.utcnow() -
datetime.datetime(1970, 1, 1, 0, 0, 0)).total_seconds())
arguments = parse_arguments()
print("\nNetCAT DNS Check, ver 5.5 - 2020, Sebastian Majewski\n")
# Setup logger
netcat.setup_logger("netcat_dnscheck",
process_name_length=15,
debug=arguments.debug)
netcat.LOGGER.info(f"Starting DNS check program, timestamp={timestamp}")
if arguments.test_run:
netcat.LOGGER.opt(ansi=True).info(
"<magenta>Test mode enabled, no information will be saved to database</magenta>"
)
if arguments.debug:
netcat.LOGGER.opt(
ansi=True).info("<magenta>Debug mode enabled</magenta>")
dns_info_list = netcat.read_info_list_file(netcat.FILENAME_DNS_INFO_LIST)
netcat.LOGGER.info(
f"Executing DNS check for {len(dns_info_list)} server(s): '{', '.join([_['ip_address'] for _ in dns_info_list])}'"
)
# Check if database tables exist, if not then create them
db.create_tables()
# Time processes execution
start_time = time.monotonic()
dns_status_document = {
"snapshot_name": "dns_status",
"snapshot_timestamp": timestamp,
}
dns_status_document["dns_data"] = await asyncio.gather(
*[dns_check(_) for _ in dns_info_list])
netcat.LOGGER.info("Saving dns status document to database")
# Cannot use netcat.exception_handler decorator due to its lack of compatibility with asyncio
try:
db.write(db.netcat.DBT_STATUS, dns_status_document)
except netcat.CustomException as exception:
netcat.LOGGER.error(f"{exception}")
sys.exit()
# Time processes execution
end_time = time.monotonic()
netcat.bind_logger("MAIN_PROG")
netcat.LOGGER.info(
f"DNS check ended, execution time: '{end_time - start_time:.2f}s'")
return 0
def create_inventory_list(device_name_list: List[str]) -> List[Dict[str, Any]]:
""" Create inventory list """
# Time process execution
start_time = time.monotonic()
# Setup logger to show process name
if os.getpid() != netcat.MAIN_PROCESS_PID:
netcat.bind_logger("SUB_PROC")
inventory_list = []
for device_data in get_device_data_list(device_name_list):
if device_data.get("device_type") == "cisco_switch":
command_output = netcat.get_command_output(device_data,
"show version")
model_numbers = netcat.find_regex_ml(command_output,
r"^Model number\s+: (\S+)$",
hint="el n",
optional=False)
serial_numbers = netcat.find_regex_ml(
command_output,
r"^System serial number\s+: (\S+)$",
hint="m s",
optional=False)
software_versions = netcat.find_regex_ml(
command_output,
r"^\*?\s+\d+\s+\d+\s+\S+\s+(\S+) .*$",
hint=" W",
optional=False)
elif device_data.get("device_type") == "cisco_router":
command_output = netcat.get_command_output(device_data,
"show version")
model_numbers = netcat.find_regex_ml(
command_output,
r"^[Cc]isco (\S+) .+ bytes of memory.$",
hint="s of m",
optional=False)
serial_numbers = netcat.find_regex_ml(
command_output,
r"^Processor board ID (\S+)$",
hint="d I",
optional=False)
software_versions = netcat.find_regex_ml(
command_output,
r"^Cisco IOS Software,? .+ Version ([^\s,]+), .+$",
hint="o I",
optional=False)
elif device_data.get("device_type") == "cisco_nexus":
command_output = netcat.get_command_output(device_data,
"show version")
model_numbers = netcat.find_regex_ml(
command_output,
r"^\s+cisco Nexus[^ ]* (\S+) .+$",
hint="has",
optional=False)
serial_numbers = netcat.find_regex_ml(
command_output,
r"^\s+Processor Board ID (\S+)$",
hint=" Pr",
optional=False)
software_versions = netcat.find_regex_ml(
command_output,
r"^\s+(?:system|NXOS):\s+version (\S+)$",
hint="ver",
optional=False)
elif device_data.get("device_type") == "cisco_asa":
command_output = netcat.get_command_output(device_data,
"show version")
model_numbers = netcat.find_regex_ml(command_output,
r"^Hardware:\s+([^ ^,]+),.+$",
hint="Har",
optional=False)
serial_numbers = netcat.find_regex_ml(command_output,
r"^Serial Number: (\S+)$",
hint="Ser",
optional=False)
software_versions = netcat.find_regex_ml(
command_output,
r"^Cisco .+ Software Version (\S+) .*$",
hint="e S",
optional=False)
elif device_data.get("device_type") == "cisco_asa_mc":
command_output = netcat.get_command_output(device_data,
"show version")
model_numbers = netcat.find_regex_ml(command_output,
r"^Hardware:\s+([^ ^,]+),.+$",
hint="Har",
optional=False)
serial_numbers = netcat.find_regex_ml(command_output,
r"^Serial Number: (\S+)$",
hint="Ser",
optional=False)
software_versions = netcat.find_regex_ml(
command_output,
r"^Cisco .+ Software Version (\S+) .*$",
hint="e S",
optional=False)
elif device_data.get("device_type") == "paloalto":
command_output = netcat.get_command_output(device_data,
"show system info")
model_numbers = netcat.find_regex_ml(command_output,
r"^model: (\S+)$",
hint="el:",
optional=False)
serial_numbers = netcat.find_regex_ml(command_output,
r"^serial: (\S+)$",
hint="ser",
optional=False)
software_versions = netcat.find_regex_ml(command_output,
r"^sw-version: (\S+)$",
hint="sw-",
optional=False)
elif device_data.get("device_type") == "f5":
command_output = netcat.get_command_output(device_data,
"show sys hardware")
model_numbers = netcat.find_regex_ml(command_output,
r"^ Name\s+(BIG-IP \S+).*$",
hint="BIG",
optional=False)
serial_numbers = netcat.find_regex_ml(
command_output,
r"^\s+Host Board Serial\s+(\S+)$",
hint=" Ho",
optional=False)
# Fix for vf2lb[12]mgmt that dont show serial numbers, can be removed after VF2 decom
if serial_numbers == []:
serial_numbers = ["UNKNOWN"]
command_output = netcat.get_command_output(device_data,
"show sys version")
software_versions = netcat.find_regex_ml(command_output,
r"^\s+Version\s+(\S+)$",
hint=" V",
optional=False)
else:
inventory_list.append(
{"device_name": device_data.get("device_name")})
continue
from uuid import uuid1
inventory_list.append({
"uuid":
uuid1(),
"snapshot_timestamp":
device_data.get("snapshot_timestamp"),
"device_name":
device_data.get("device_name"),
"device_type":
device_data.get("device_type"),
"chasis": [{
"model": _,
"serial": __,
"software": ___
} for _, __, ___ in zip(model_numbers, serial_numbers,
software_versions)],
})
# Time process execution
end_time = time.monotonic()
netcat.LOGGER.debug(
f"Inventory data created for {len(device_name_list)} devices in {end_time - start_time:.2f}s"
)
return inventory_list
def find_broken_links_per_device(
device_data: Dict[str, Any]) -> List[Dict[str, Any]]:
""" Search for any link that is down (but not admin down) on Cisco routers and Palo Alto firewalls """
# Time process execution
start_time = time.monotonic()
# Setup logger to show process name
if os.getpid() != netcat.MAIN_PROCESS_PID:
netcat.bind_logger("SUB_PROC")
broken_links = []
if device_data.get("device_type") == "cisco_router":
# Look for any broken link
for interface_name, interface_ip_address in netcat.find_regex_ml(
netcat.get_command_output(device_data,
"show ip interface brief"),
rf"^([^\s]*(?:Ethernet|Tunnel)\S+)\s+(\S+)\s+\S+\s+\S+\s+(?:up|down)\s+down\s*$"
):
broken_link = {
"uuid": uuid.uuid1(),
"device_name": device_data.get("device_name"),
"device_type": device_data.get("device_type"),
"interface_name": interface_name,
"interface_name_encoded": interface_name.replace("/", "_"),
"interface_ip_address": interface_ip_address,
}
# Search for latest device_info structure that has broken link in UP state and record its timestamp
device_data_list = db.get_device_data_list__c(
device_data.get("device_name", ""),
command_list=["show ip interface brief"])
for device_data in device_data_list:
regex_interface_name = interface_name.replace("/",
"\\/").replace(
".", "\.")
if netcat.find_regex_sl(
netcat.get_command_output(device_data,
"show ip interface brief"),
rf"(^[^\s]*{regex_interface_name}\s+\S+\s+\S+\s+\S+\s+up\s+up\s*$)",
hint=interface_name,
optional=False):
broken_link["snapshot_timestamp"] = device_data.get(
"snapshot_timestamp")
break
broken_links.append(broken_link)
elif device_data.get("device_type") == "paloalto":
# Skip device if its not in active ha state
if netcat.find_regex_sl(
netcat.get_command_output(device_data,
"show high-availability all"),
r"\s+State: (\S+) .*$") not in {
"active", "active-primary", "active-secondary", ""
}:
return []
# Look for any broken link
for interface_name, interface_mac_address in netcat.find_regex_ml(
netcat.get_command_output(device_data, "show interface all"),
r"^((?:ethernet|ae)\S+)\s+\d+\s+ukn\/ukn\/down\S+\s+(\S+)\s*$"
):
broken_link = {
"uuid": uuid.uuid1(),
"device_name": device_data.get("device_name"),
"device_type": device_data.get("device_type"),
"interface_name": interface_name,
"interface_name_encoded": interface_name.replace("/", "_"),
"interface_ip_address": "N/A",
"interface_mac_address": interface_mac_address,
}
# Search for latest device_info structure that has broken BGP session in UP state and record its timestamp
device_data_list = db.get_device_data_list__c(
device_data.get("device_name", ""),
command_list=["show interface all"])
for device_data in device_data_list:
regex_interface_name = interface_name.replace("/",
"\\/").replace(
".", "\.")
if netcat.find_regex_ml(
netcat.get_command_output(device_data,
"show interface all"),
rf"(^{regex_interface_name}\s+\S+\s+\S+\/up\s+\S+\s*$)",
hint=interface_name,
optional=False):
broken_link["snapshot_timestamp"] = device_data.get(
"snapshot_timestamp")
break
broken_links.append(broken_link)
else:
netcat.LOGGER.warning(
f"{netcat.fn()}: Unknown device data type value '{device_data.get('type')}'"
)
# Time process execution
end_time = time.monotonic()
netcat.LOGGER.debug(
f"Created broken links list of {len(broken_links)} links in {end_time - start_time:.2f}s"
)
return broken_links
def find_broken_bgp_sessions_per_device(
device_data: Dict[str, Any]) -> List[Dict[str, Any]]:
""" Search for broken BGP sessions on Cisco routers and Palo Alto firewalls """
# Time process execution
start_time = time.monotonic()
# Setup logger to show process name
if os.getpid() != netcat.MAIN_PROCESS_PID:
netcat.bind_logger("SUB_PROC")
broken_bgp_sessions = []
if device_data.get("device_type") == "cisco_router":
# Look for any broken BGP sessions
for bgp_session_peer_ip, bgp_session_peer_asn in netcat.find_regex_ml(
netcat.get_command_output(device_data, "show ip bgp summary"),
r"^(\S+)\s+\d\s+(\d+)\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+\S+\s+(?:Idle|Active)$"
):
broken_bgp_session = {
"uuid": uuid.uuid1(),
"device_name": device_data.get("device_name"),
"device_type": device_data.get("device_type"),
"peer_ip": bgp_session_peer_ip,
"peer_asn": bgp_session_peer_asn,
}
# Search for latest device_data document that has broken BGP session in UP state and record its timestamp
device_data_list = db.get_device_data_list__c(
device_data.get("device_name", ""),
command_list=["show ip bgp summary"])
for device_data in device_data_list:
if netcat.find_regex_ml(
netcat.get_command_output(device_data,
"show ip bgp summary"),
rf"(^{bgp_session_peer_ip}\s+\d\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+\S+\s+\d+$)",
hint=bgp_session_peer_ip,
optional=False):
broken_bgp_session["snapshot_timestamp"] = device_data.get(
"snapshot_timestamp")
break
broken_bgp_sessions.append(broken_bgp_session)
elif device_data.get("device_type") == "paloalto":
# Skip device if its not in active ha state
if netcat.find_regex_sl(
netcat.get_command_output(device_data,
"show high-availability all"),
r"\s+State: (\S+) .*$") not in {
"active", "active-primary", "active-secondary", ""
}:
return []
# Look for any broken BGP sessions
for bgp_session_peer_asn, bgp_session_peer_ip in netcat.find_regex_ml(
netcat.get_command_output(device_data,
"show routing protocol bgp summary"),
r"^\s+peer \S+\s+ AS (\d+), (?:Connect|Active), IP (\S+)$"):
broken_bgp_session = {
"uuid": uuid.uuid1(),
"device_name": device_data.get("device_name"),
"device_type": device_data.get("device_type"),
"peer_ip": bgp_session_peer_ip,
"peer_asn": bgp_session_peer_asn,
}
# Search for latest device_data structure that has broken BGP session in UP state and record its timestamp
device_data_list = db.get_device_data_list__c(
device_data.get("device_name", ""),
command_list=[
"show routing protocol bgp summary",
"show high-availability all"
])
for device_data in device_data_list:
if netcat.find_regex_sl(
netcat.get_command_output(
device_data, "show routing protocol bgp summary"),
rf"(^\s+peer \S+\s+ AS \d+, Established, IP {bgp_session_peer_ip})$",
hint=bgp_session_peer_ip,
optional=False):
broken_bgp_session["snapshot_timestamp"] = device_data.get(
"snapshot_timestamp")
break
broken_bgp_sessions.append(broken_bgp_session)
else:
netcat.LOGGER.warning(
f"{netcat.fn()}: Unknown device data type value '{device_data.get('type')}'"
)
# Time process execution
end_time = time.monotonic()
netcat.LOGGER.debug(
f"Created broken bgp session list of {len(broken_bgp_sessions)} sessions in {end_time - start_time:.2f}s"
)
return broken_bgp_sessions
