def cli_process(device_info: Dict[str, str], timestamp: int, upgrade: bool) -> List[str]: """ Command line process, ready to run as separate thread or process """ # Setup logger to show process name netcat.bind_logger(device_info["device_name"].upper()) # Time process execution start_time = time.time() # Log initial status if netcat.SINGLE_PROCESS_MODE: netcat.LOGGER.opt(ansi=True).info( "<green>Executing CLI operations as part of main process </green>") else: netcat.LOGGER.opt(ansi=True).info( "<green>Executing CLI operations as child process</green>") upgrade_software(device_info, REQUESTED_SOFTWARE_VERSION, upgrade) # Time process execution end_time = time.time() # Log process end status and execution time netcat.LOGGER.opt(ansi=True).info( f"<green>CLI process ended normaly, execution time: {end_time - start_time:.2f}s</green>" ) # Return device name to indicate successfuly executed operation for given device return [device_info["device_name"]]
def get_failed_device_data_list( device_list: List[Tuple[str, str]]) -> List[Dict[str, Any]]: """ Prepare list of device data documents to be used to create failed device data list """ # Time process execution start_time = time.monotonic() # Setup logger to show process name if os.getpid() != netcat.MAIN_PROCESS_PID: netcat.bind_logger("SUB_PROC") # Pull list of latest device_data documents for devices in device_list device_data_list = db.get_device_data_list__b([_[0] for _ in device_list], command_list=None) # Create mock entries for devices that exist in command snapshot status but were never pooled successfuly for device_name in set([_[0] for _ in device_list]) - set( _.get("device_name") for _ in device_data_list): device_data_list.append({ "device_name": device_name, "device_type": next(iter([_[1] for _ in device_list if _[0] == device_name]), None) }) # Time process execution end_time = time.monotonic() netcat.LOGGER.debug( f"Created inaccessible device list for {len(device_list)} devices in {end_time - start_time:.2f}s" ) return device_data_list
def cli_process(device_info: Dict[str, str], timestamp: int, force_backup: bool = False, test_run: bool = False) -> List[str]: """ Command line process, ready to run as separate thread or process """ # Setup logger to show process name netcat.bind_logger(device_info["device_name"].upper()) # Time process execution start_time = time.time() # Log initial status if netcat.SINGLE_PROCESS_MODE: netcat.LOGGER.opt(ansi=True).info("<green>Executing CLI operations as part of main process </green>") else: netcat.LOGGER.opt(ansi=True).info("<green>Executing CLI operations as child process</green>") # Access device and retrieve its current configuration in all formats, update existing device_dat structure device_data = get_device_data(device_info) if not test_run: # Get list of previous configuration backups stored localy and compare latest of them with current config set config_change = detect_config_change(device_data) # Save device info to databae save_device_data(device_data, timestamp, config_change, force_backup) # Time process execution end_time = time.time() # Log process end status and execution time netcat.LOGGER.opt(ansi=True).info(f"<green>CLI process ended normaly, execution time: {end_time - start_time:.2f}s</green>") # Return device name to indicate successfuly executed operation for given device return [device_info["device_name"]]
def find_mac_address_in_snapshot(timestamp: int, mac_address: str, physical_ports_only: bool) -> List[Dict[str, str]]: """ Execute 'find_ip_address_in_device_data()' function on all devices info files with given timestamp """ # Time process execution start_time = time.monotonic() netcat.bind_logger("SUB_PROC") device_data_list = db.get_device_data_list__a(timestamp, ["cisco_switch", "cisco_nexus"], command_list=["show mac address-table"]) findings = [_ for __ in device_data_list for _ in find_mac_address_in_device_data(__, mac_address, physical_ports_only)] # Time process execution end_time = time.monotonic() netcat.LOGGER.debug(f"Search for '{mac_address}' performed on {len(device_data_list)} devices in '{timestamp}' snapshot in {end_time - start_time:.2f}s") return findings
def find_ip_address_in_snapshot(timestamp: int, ip_address: str, use_arp: bool, use_dhcp: bool, use_dsnp: bool) -> List[Dict[str, str]]: """ Execute 'find_ip_address_in_device_data()' function on all devices info files with given timestamp """ # Time process execution start_time = time.monotonic() netcat.bind_logger("SUB_PROC") device_type_list: Set[str] = set() command_list: Set[str] = set() if use_arp: device_type_list |= {"cisco_router", "paloalto"} command_list |= {"show ip arp", "show arp all"} if use_dhcp: device_type_list |= {"cisco_router", "paloalto"} command_list |= { "show ip dhcp binding", "show dhcp server lease interface all" } if use_dsnp: device_type_list |= {"cisco_switch"} command_list |= {"show ip dhcp snooping binding"} device_data_list = db.get_device_data_list__a(timestamp, list(device_type_list), list(command_list)) findings = [ _ for __ in device_data_list for _ in find_ip_address_in_device_data( __, ip_address, use_arp, use_dhcp, use_dsnp) ] # Time process execution end_time = time.monotonic() netcat.LOGGER.debug( f"Search for '{ip_address}' performed on {len(device_data_list)} devices in '{timestamp}' snapshot in {end_time - start_time:.2f}s" ) return findings
async def _(hostname): try: await resolver.query(hostname, "A") except aiodns.error.DNSError as exception: if exception.args[0] == 4: netcat.bind_logger(dns_info["description"]) netcat.LOGGER.info(f"Not able to resolve '{hostname}'") return "FAIL [R]" if exception.args[0] == 12: netcat.bind_logger(dns_info["description"]) netcat.LOGGER.info( f"Not able to connect to server trying to resolve '{hostname}'" ) return "FAIL [C]" netcat.LOGGER.info( f"Unknown error trying to resolve '{hostname}' - '{exception.args[1]}'" ) return "FAIL [U]" netcat.bind_logger(dns_info["description"]) netcat.LOGGER.info(f"Successfully resolved '{hostname}'") return "OK [CR]"
def cli_process(device_info: Dict[str, str], snippet: str, site_id_check: bool, inet_gw_check: bool, no_commit: bool) -> List[Dict[str, str]]: """ Command line process, ready to run as separate thread or process """ # Setup logger to show process name netcat.bind_logger(device_info["device_name"].upper()) # Time process execution start_time = time.time() # Log initial status netcat.LOGGER.opt(ansi=True).info("<green>Starting CLI process</green>") # Deploy configuration snippet deploy_config_snippet(device_info, snippet, site_id_check, inet_gw_check, no_commit) # Time process execution end_time = time.time() # Log process end status and execution time netcat.LOGGER.opt(ansi=True).info(f"<green>CLI process ended normaly, execution time: {end_time - start_time:.2f}s</green>") return [device_info["device_name"]]
async def dns_check(dns_info: Dict[str, str]) -> None: """ Perform DNS check """ dns_data = { "description": dns_info["description"], "ip_address": dns_info["ip_address"], "results": {} } async def _(hostname): try: await resolver.query(hostname, "A") except aiodns.error.DNSError as exception: if exception.args[0] == 4: netcat.bind_logger(dns_info["description"]) netcat.LOGGER.info(f"Not able to resolve '{hostname}'") return "FAIL [R]" if exception.args[0] == 12: netcat.bind_logger(dns_info["description"]) netcat.LOGGER.info( f"Not able to connect to server trying to resolve '{hostname}'" ) return "FAIL [C]" netcat.LOGGER.info( f"Unknown error trying to resolve '{hostname}' - '{exception.args[1]}'" ) return "FAIL [U]" netcat.bind_logger(dns_info["description"]) netcat.LOGGER.info(f"Successfully resolved '{hostname}'") return "OK [CR]" resolver = aiodns.DNSResolver(timeout=0) resolver.nameservers = [dns_info["ip_address"]] netcat.bind_logger(dns_info["description"]) netcat.LOGGER.info(f"Querrying server for '{HOSTNAME_EXTERNAL}'") dns_data["results"]["external"] = await _(HOSTNAME_EXTERNAL) netcat.bind_logger(dns_info["description"]) netcat.LOGGER.info(f"Querrying server for '{HOSTNAME_INTERNAL}'") dns_data["results"]["internal"] = await _(HOSTNAME_INTERNAL) netcat.bind_logger("MAIN_PROG") return dns_data
async def main() -> int: """ Main program """ timestamp = int((datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1, 0, 0, 0)).total_seconds()) arguments = parse_arguments() print("\nNetCAT DNS Check, ver 5.5 - 2020, Sebastian Majewski\n") # Setup logger netcat.setup_logger("netcat_dnscheck", process_name_length=15, debug=arguments.debug) netcat.LOGGER.info(f"Starting DNS check program, timestamp={timestamp}") if arguments.test_run: netcat.LOGGER.opt(ansi=True).info( "<magenta>Test mode enabled, no information will be saved to database</magenta>" ) if arguments.debug: netcat.LOGGER.opt( ansi=True).info("<magenta>Debug mode enabled</magenta>") dns_info_list = netcat.read_info_list_file(netcat.FILENAME_DNS_INFO_LIST) netcat.LOGGER.info( f"Executing DNS check for {len(dns_info_list)} server(s): '{', '.join([_['ip_address'] for _ in dns_info_list])}'" ) # Check if database tables exist, if not then create them db.create_tables() # Time processes execution start_time = time.monotonic() dns_status_document = { "snapshot_name": "dns_status", "snapshot_timestamp": timestamp, } dns_status_document["dns_data"] = await asyncio.gather( *[dns_check(_) for _ in dns_info_list]) netcat.LOGGER.info("Saving dns status document to database") # Cannot use netcat.exception_handler decorator due to its lack of compatibility with asyncio try: db.write(db.netcat.DBT_STATUS, dns_status_document) except netcat.CustomException as exception: netcat.LOGGER.error(f"{exception}") sys.exit() # Time processes execution end_time = time.monotonic() netcat.bind_logger("MAIN_PROG") netcat.LOGGER.info( f"DNS check ended, execution time: '{end_time - start_time:.2f}s'") return 0
def create_inventory_list(device_name_list: List[str]) -> List[Dict[str, Any]]: """ Create inventory list """ # Time process execution start_time = time.monotonic() # Setup logger to show process name if os.getpid() != netcat.MAIN_PROCESS_PID: netcat.bind_logger("SUB_PROC") inventory_list = [] for device_data in get_device_data_list(device_name_list): if device_data.get("device_type") == "cisco_switch": command_output = netcat.get_command_output(device_data, "show version") model_numbers = netcat.find_regex_ml(command_output, r"^Model number\s+: (\S+)$", hint="el n", optional=False) serial_numbers = netcat.find_regex_ml( command_output, r"^System serial number\s+: (\S+)$", hint="m s", optional=False) software_versions = netcat.find_regex_ml( command_output, r"^\*?\s+\d+\s+\d+\s+\S+\s+(\S+) .*$", hint=" W", optional=False) elif device_data.get("device_type") == "cisco_router": command_output = netcat.get_command_output(device_data, "show version") model_numbers = netcat.find_regex_ml( command_output, r"^[Cc]isco (\S+) .+ bytes of memory.$", hint="s of m", optional=False) serial_numbers = netcat.find_regex_ml( command_output, r"^Processor board ID (\S+)$", hint="d I", optional=False) software_versions = netcat.find_regex_ml( command_output, r"^Cisco IOS Software,? .+ Version ([^\s,]+), .+$", hint="o I", optional=False) elif device_data.get("device_type") == "cisco_nexus": command_output = netcat.get_command_output(device_data, "show version") model_numbers = netcat.find_regex_ml( command_output, r"^\s+cisco Nexus[^ ]* (\S+) .+$", hint="has", optional=False) serial_numbers = netcat.find_regex_ml( command_output, r"^\s+Processor Board ID (\S+)$", hint=" Pr", optional=False) software_versions = netcat.find_regex_ml( command_output, r"^\s+(?:system|NXOS):\s+version (\S+)$", hint="ver", optional=False) elif device_data.get("device_type") == "cisco_asa": command_output = netcat.get_command_output(device_data, "show version") model_numbers = netcat.find_regex_ml(command_output, r"^Hardware:\s+([^ ^,]+),.+$", hint="Har", optional=False) serial_numbers = netcat.find_regex_ml(command_output, r"^Serial Number: (\S+)$", hint="Ser", optional=False) software_versions = netcat.find_regex_ml( command_output, r"^Cisco .+ Software Version (\S+) .*$", hint="e S", optional=False) elif device_data.get("device_type") == "cisco_asa_mc": command_output = netcat.get_command_output(device_data, "show version") model_numbers = netcat.find_regex_ml(command_output, r"^Hardware:\s+([^ ^,]+),.+$", hint="Har", optional=False) serial_numbers = netcat.find_regex_ml(command_output, r"^Serial Number: (\S+)$", hint="Ser", optional=False) software_versions = netcat.find_regex_ml( command_output, r"^Cisco .+ Software Version (\S+) .*$", hint="e S", optional=False) elif device_data.get("device_type") == "paloalto": command_output = netcat.get_command_output(device_data, "show system info") model_numbers = netcat.find_regex_ml(command_output, r"^model: (\S+)$", hint="el:", optional=False) serial_numbers = netcat.find_regex_ml(command_output, r"^serial: (\S+)$", hint="ser", optional=False) software_versions = netcat.find_regex_ml(command_output, r"^sw-version: (\S+)$", hint="sw-", optional=False) elif device_data.get("device_type") == "f5": command_output = netcat.get_command_output(device_data, "show sys hardware") model_numbers = netcat.find_regex_ml(command_output, r"^ Name\s+(BIG-IP \S+).*$", hint="BIG", optional=False) serial_numbers = netcat.find_regex_ml( command_output, r"^\s+Host Board Serial\s+(\S+)$", hint=" Ho", optional=False) # Fix for vf2lb[12]mgmt that dont show serial numbers, can be removed after VF2 decom if serial_numbers == []: serial_numbers = ["UNKNOWN"] command_output = netcat.get_command_output(device_data, "show sys version") software_versions = netcat.find_regex_ml(command_output, r"^\s+Version\s+(\S+)$", hint=" V", optional=False) else: inventory_list.append( {"device_name": device_data.get("device_name")}) continue from uuid import uuid1 inventory_list.append({ "uuid": uuid1(), "snapshot_timestamp": device_data.get("snapshot_timestamp"), "device_name": device_data.get("device_name"), "device_type": device_data.get("device_type"), "chasis": [{ "model": _, "serial": __, "software": ___ } for _, __, ___ in zip(model_numbers, serial_numbers, software_versions)], }) # Time process execution end_time = time.monotonic() netcat.LOGGER.debug( f"Inventory data created for {len(device_name_list)} devices in {end_time - start_time:.2f}s" ) return inventory_list
def find_broken_links_per_device( device_data: Dict[str, Any]) -> List[Dict[str, Any]]: """ Search for any link that is down (but not admin down) on Cisco routers and Palo Alto firewalls """ # Time process execution start_time = time.monotonic() # Setup logger to show process name if os.getpid() != netcat.MAIN_PROCESS_PID: netcat.bind_logger("SUB_PROC") broken_links = [] if device_data.get("device_type") == "cisco_router": # Look for any broken link for interface_name, interface_ip_address in netcat.find_regex_ml( netcat.get_command_output(device_data, "show ip interface brief"), rf"^([^\s]*(?:Ethernet|Tunnel)\S+)\s+(\S+)\s+\S+\s+\S+\s+(?:up|down)\s+down\s*$" ): broken_link = { "uuid": uuid.uuid1(), "device_name": device_data.get("device_name"), "device_type": device_data.get("device_type"), "interface_name": interface_name, "interface_name_encoded": interface_name.replace("/", "_"), "interface_ip_address": interface_ip_address, } # Search for latest device_info structure that has broken link in UP state and record its timestamp device_data_list = db.get_device_data_list__c( device_data.get("device_name", ""), command_list=["show ip interface brief"]) for device_data in device_data_list: regex_interface_name = interface_name.replace("/", "\\/").replace( ".", "\.") if netcat.find_regex_sl( netcat.get_command_output(device_data, "show ip interface brief"), rf"(^[^\s]*{regex_interface_name}\s+\S+\s+\S+\s+\S+\s+up\s+up\s*$)", hint=interface_name, optional=False): broken_link["snapshot_timestamp"] = device_data.get( "snapshot_timestamp") break broken_links.append(broken_link) elif device_data.get("device_type") == "paloalto": # Skip device if its not in active ha state if netcat.find_regex_sl( netcat.get_command_output(device_data, "show high-availability all"), r"\s+State: (\S+) .*$") not in { "active", "active-primary", "active-secondary", "" }: return [] # Look for any broken link for interface_name, interface_mac_address in netcat.find_regex_ml( netcat.get_command_output(device_data, "show interface all"), r"^((?:ethernet|ae)\S+)\s+\d+\s+ukn\/ukn\/down\S+\s+(\S+)\s*$" ): broken_link = { "uuid": uuid.uuid1(), "device_name": device_data.get("device_name"), "device_type": device_data.get("device_type"), "interface_name": interface_name, "interface_name_encoded": interface_name.replace("/", "_"), "interface_ip_address": "N/A", "interface_mac_address": interface_mac_address, } # Search for latest device_info structure that has broken BGP session in UP state and record its timestamp device_data_list = db.get_device_data_list__c( device_data.get("device_name", ""), command_list=["show interface all"]) for device_data in device_data_list: regex_interface_name = interface_name.replace("/", "\\/").replace( ".", "\.") if netcat.find_regex_ml( netcat.get_command_output(device_data, "show interface all"), rf"(^{regex_interface_name}\s+\S+\s+\S+\/up\s+\S+\s*$)", hint=interface_name, optional=False): broken_link["snapshot_timestamp"] = device_data.get( "snapshot_timestamp") break broken_links.append(broken_link) else: netcat.LOGGER.warning( f"{netcat.fn()}: Unknown device data type value '{device_data.get('type')}'" ) # Time process execution end_time = time.monotonic() netcat.LOGGER.debug( f"Created broken links list of {len(broken_links)} links in {end_time - start_time:.2f}s" ) return broken_links
def find_broken_bgp_sessions_per_device( device_data: Dict[str, Any]) -> List[Dict[str, Any]]: """ Search for broken BGP sessions on Cisco routers and Palo Alto firewalls """ # Time process execution start_time = time.monotonic() # Setup logger to show process name if os.getpid() != netcat.MAIN_PROCESS_PID: netcat.bind_logger("SUB_PROC") broken_bgp_sessions = [] if device_data.get("device_type") == "cisco_router": # Look for any broken BGP sessions for bgp_session_peer_ip, bgp_session_peer_asn in netcat.find_regex_ml( netcat.get_command_output(device_data, "show ip bgp summary"), r"^(\S+)\s+\d\s+(\d+)\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+\S+\s+(?:Idle|Active)$" ): broken_bgp_session = { "uuid": uuid.uuid1(), "device_name": device_data.get("device_name"), "device_type": device_data.get("device_type"), "peer_ip": bgp_session_peer_ip, "peer_asn": bgp_session_peer_asn, } # Search for latest device_data document that has broken BGP session in UP state and record its timestamp device_data_list = db.get_device_data_list__c( device_data.get("device_name", ""), command_list=["show ip bgp summary"]) for device_data in device_data_list: if netcat.find_regex_ml( netcat.get_command_output(device_data, "show ip bgp summary"), rf"(^{bgp_session_peer_ip}\s+\d\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+\S+\s+\d+$)", hint=bgp_session_peer_ip, optional=False): broken_bgp_session["snapshot_timestamp"] = device_data.get( "snapshot_timestamp") break broken_bgp_sessions.append(broken_bgp_session) elif device_data.get("device_type") == "paloalto": # Skip device if its not in active ha state if netcat.find_regex_sl( netcat.get_command_output(device_data, "show high-availability all"), r"\s+State: (\S+) .*$") not in { "active", "active-primary", "active-secondary", "" }: return [] # Look for any broken BGP sessions for bgp_session_peer_asn, bgp_session_peer_ip in netcat.find_regex_ml( netcat.get_command_output(device_data, "show routing protocol bgp summary"), r"^\s+peer \S+\s+ AS (\d+), (?:Connect|Active), IP (\S+)$"): broken_bgp_session = { "uuid": uuid.uuid1(), "device_name": device_data.get("device_name"), "device_type": device_data.get("device_type"), "peer_ip": bgp_session_peer_ip, "peer_asn": bgp_session_peer_asn, } # Search for latest device_data structure that has broken BGP session in UP state and record its timestamp device_data_list = db.get_device_data_list__c( device_data.get("device_name", ""), command_list=[ "show routing protocol bgp summary", "show high-availability all" ]) for device_data in device_data_list: if netcat.find_regex_sl( netcat.get_command_output( device_data, "show routing protocol bgp summary"), rf"(^\s+peer \S+\s+ AS \d+, Established, IP {bgp_session_peer_ip})$", hint=bgp_session_peer_ip, optional=False): broken_bgp_session["snapshot_timestamp"] = device_data.get( "snapshot_timestamp") break broken_bgp_sessions.append(broken_bgp_session) else: netcat.LOGGER.warning( f"{netcat.fn()}: Unknown device data type value '{device_data.get('type')}'" ) # Time process execution end_time = time.monotonic() netcat.LOGGER.debug( f"Created broken bgp session list of {len(broken_bgp_sessions)} sessions in {end_time - start_time:.2f}s" ) return broken_bgp_sessions