def download_software(self, requested_software_version: str) -> None:
""" Download software """
major = requested_software_version.split(".")[0]
minor = requested_software_version.split(".")[1]
patch = requested_software_version.split(".")[2]
requested_software_version_dependencies = [f"{major}.0.0"]
if minor != "0":
requested_software_version_dependencies.append(
f"{major}.{minor}.0")
if patch != "0":
requested_software_version_dependencies.append(
f"{major}.{minor}.{patch}")
netcat.LOGGER.info(
"Detected software version dependencies: {}",
" -> ".join(requested_software_version_dependencies))
# Download latest software list
netcat.LOGGER.info("Refreshing available software versions")
available_software_versions = self.send_command(
"request system software check", timeout=120)
if server_error := netcat.find_regex_sl(available_software_versions,
r"(Server error)"):
raise netcat.CustomException(f"Received: '{server_error}'")
def get_site_id(self) -> str:
""" Detect site ID """
if site_id := netcat.find_regex_sl(
self.send_command("show routing protocol bgp summary"),
r"^ +router id: +\d+\.(\d+)\.\d+\.\d+$"):
netcat.LOGGER.info(f"Detected Site ID: {site_id}")
return site_id
def display_resource_availability_history(device_name: str, resource: str,
resource_name: str,
cisco_command: str, cisco_regex: str,
pa_regex: str,
pa_command: str) -> str:
""" Display resource history for given device name and resource """
# Time process execution
start_time = time.monotonic()
timestamp_list = [
_.get("snapshot_timestamp") for _ in db.get_command_status_list()
if type(_) is dict
]
device_data_list = db.get_device_data_list__c(device_name,
[pa_command, cisco_command])
availability_history = []
for timestamp in timestamp_list:
device_data: Optional[Dict[str, Any]] = next(
(_ for _ in device_data_list
if _.get("snapshot_timestamp") == timestamp), None)
if device_data is None:
availability_history.append({
"uuid": uuid.uuid1(),
"snapshot_timestamp": timestamp
})
continue
if device_data.get("device_type") == "cisco_router":
if status := netcat.find_regex_sl(
netcat.get_command_output(device_data, cisco_command),
cisco_regex):
availability_history.append({
"uuid": uuid.uuid1(),
"snapshot_timestamp": timestamp,
"status": status
})
continue
elif device_data.get("device_type") == "paloalto":
if status := netcat.find_regex_sl(
netcat.get_command_output(device_data, pa_command),
pa_regex):
availability_history.append({
"uuid": uuid.uuid1(),
"snapshot_timestamp": timestamp,
"status": status
})
continue
def get_site_id(self) -> str:
""" Detect site ID """
netcat.LOGGER.info("Detecting Site ID")
# We do it for routers only, makes little sense to do for other devices
if self.type == "cisco_router":
if site_id := netcat.find_regex_sl(
self.send_command("show ip bgp summary"),
r"^BGP router identifier \d+\.(\d+).\d+.\d+,.*$"):
netcat.LOGGER.info(f"Detected Site ID: {site_id}")
return site_id
def get_inet_gw(self) -> str:
""" Detect Internet default gateway """
netcat.LOGGER.info("Detecting Internet default gateway IP address")
if inet_gw := netcat.find_regex_sl(
self.send_config_command(
"show network virtual-router VR_GLOBAL routing-table ip static-route SR_DEFAULT nexthop"
), r"^.+ (\d+\.\d+\.\d+.\d+)$"):
netcat.LOGGER.info(
f"Detected Internet default IP address: {inet_gw}")
return inet_gw
def get_inet_gw(self) -> str:
""" Detect Internet default gateway """
netcat.LOGGER.info("Detecting Internet default gateway IP address")
# We do it for routers only, makes little sense to do for other devices
if self.type == "cisco_router":
if inet_gw := netcat.find_regex_sl(
self.send_command(
"show running-config | include 0.0.0.0 0.0.0.0"),
r"^ip route (?:vrf INTERNET )?0\.0\.0\.0 0\.0\.0\.0 (\d+\.\d+\.\d+\.\d+) .*$"
):
netcat.LOGGER.info(
"Detected Internet default gateway IP address: {}",
inet_gw)
return inet_gw
def clear_commit_in_progress(self) -> None:
""" Check if there is any commit in progress and wait till finishes or time out after 3 minutes"""
netcat.LOGGER.info("Checking for any other commit in progress")
for _ in range(6):
if netcat.find_regex_sl(
self.send_command("show jobs processed"),
r"(^[^ ]+ [^ ]+ +[^ ]+ +\d+ +Commit +ACT .*$)"):
netcat.LOGGER.warning(
"Another commit in progress, will wait 30s and recheck")
time.sleep(30)
continue
break
else:
raise netcat.CustomException(
"Another commit in progress takes over 3 minutes")
netcat.LOGGER.info("No other commit in progress")
def upgrade_software(self, requested_software_version: str) -> None:
""" Upgrade software """
# Make up to three attempts to install software
for _ in range(3):
for _ in range(30):
command_output = self.send_command(
f"request system software install version {requested_software_version}"
)
if netcat.find_regex_sl(command_output, r"(Server error)"):
if netcat.find_regex_sl(command_output,
r"(install is in progress)"):
netcat.LOGGER.info(
"Another installation in progress, waiting...")
time.sleep(10)
continue
if netcat.find_regex_sl(
command_output,
r"(pending jobs in the commit task queue)"):
netcat.LOGGER.info(
"Pending jobs in commit task queue, waiting...")
time.sleep(10)
continue
if netcat.find_regex_sl(command_output,
r"(commit is in progress)"):
netcat.LOGGER.info("Commit is in progress, waiting...")
time.sleep(10)
continue
raise netcat.CustomException(
f"Received: '{netcat.find_regex_sl(command_output, r'(Server error)')}'"
)
break
else:
raise netcat.CustomException(
"Another installation in progress for over 5 minutes")
job_id = netcat.find_regex_sl(
command_output,
r"^Software install job enqueued with jobid (\d+)\.\s+.*$")
netcat.LOGGER.info(
f"Installation of software version {requested_software_version} started with job id '{job_id}'"
)
time.sleep(5)
while ((
command_output :=
self.send_command(f"show jobs id {job_id}")
) and netcat.find_regex_sl(
command_output,
rf"^\d\S+\s+\S+\s+(?:\S+\s+)?\d+\s+SWInstall\s+(\S+)\s+\S+\s+\S+\s*$"
) in {"ACT", "QUEUED"}):
installation_progress = netcat.find_regex_sl(
command_output,
rf"^\d\S+\s+\S+\s+\S+\s+\d+\s+SWInstall\s+\S+\s+\S+\s+(\S+)\s*$"
)
netcat.LOGGER.info(
f"Installing software version {requested_software_version}, progress {installation_progress}"
)
time.sleep(5)
if netcat.find_regex_sl(
command_output,
rf"^\d\S+\s+\S+\s+\S+\s+\d+\s+SWInstall\s+FIN\s+(\S+)\s+\S+\s*$"
) == "OK":
netcat.LOGGER.info(
f"Installation of software version {requested_software_version} completed"
)
break
netcat.LOGGER.warning(
f"Installation of version {requested_software_version} failed, will retry up to three times..."
)
print("***", command_output, "***")
netcat.LOGGER.info(
"Detected software version dependencies: {}",
" -> ".join(requested_software_version_dependencies))
# Download latest software list
netcat.LOGGER.info("Refreshing available software versions")
available_software_versions = self.send_command(
"request system software check", timeout=120)
if server_error := netcat.find_regex_sl(available_software_versions,
r"(Server error)"):
raise netcat.CustomException(f"Received: '{server_error}'")
# Download all the software versions from dependency list
for software_version_dependency in requested_software_version_dependencies:
if netcat.find_regex_sl(
available_software_versions,
rf"^{software_version_dependency}\s+\S+\s+\S+\s+\S+\s+(\S+)\s*$"
) == "yes":
netcat.LOGGER.info(
f"Software version {software_version_dependency} already downloaded"
)
continue
# Make up to three attempts to download software
for _ in range(3):
netcat.LOGGER.info(
f"Attempting to download software version: {software_version_dependency}"
)
# Wait up to five minutes in case any other download is in progress
for _ in range(30):
command_output = self.send_command(
def find_broken_links_per_device(
device_data: Dict[str, Any]) -> List[Dict[str, Any]]:
""" Search for any link that is down (but not admin down) on Cisco routers and Palo Alto firewalls """
# Time process execution
start_time = time.monotonic()
# Setup logger to show process name
if os.getpid() != netcat.MAIN_PROCESS_PID:
netcat.bind_logger("SUB_PROC")
broken_links = []
if device_data.get("device_type") == "cisco_router":
# Look for any broken link
for interface_name, interface_ip_address in netcat.find_regex_ml(
netcat.get_command_output(device_data,
"show ip interface brief"),
rf"^([^\s]*(?:Ethernet|Tunnel)\S+)\s+(\S+)\s+\S+\s+\S+\s+(?:up|down)\s+down\s*$"
):
broken_link = {
"uuid": uuid.uuid1(),
"device_name": device_data.get("device_name"),
"device_type": device_data.get("device_type"),
"interface_name": interface_name,
"interface_name_encoded": interface_name.replace("/", "_"),
"interface_ip_address": interface_ip_address,
}
# Search for latest device_info structure that has broken link in UP state and record its timestamp
device_data_list = db.get_device_data_list__c(
device_data.get("device_name", ""),
command_list=["show ip interface brief"])
for device_data in device_data_list:
regex_interface_name = interface_name.replace("/",
"\\/").replace(
".", "\.")
if netcat.find_regex_sl(
netcat.get_command_output(device_data,
"show ip interface brief"),
rf"(^[^\s]*{regex_interface_name}\s+\S+\s+\S+\s+\S+\s+up\s+up\s*$)",
hint=interface_name,
optional=False):
broken_link["snapshot_timestamp"] = device_data.get(
"snapshot_timestamp")
break
broken_links.append(broken_link)
elif device_data.get("device_type") == "paloalto":
# Skip device if its not in active ha state
if netcat.find_regex_sl(
netcat.get_command_output(device_data,
"show high-availability all"),
r"\s+State: (\S+) .*$") not in {
"active", "active-primary", "active-secondary", ""
}:
return []
# Look for any broken link
for interface_name, interface_mac_address in netcat.find_regex_ml(
netcat.get_command_output(device_data, "show interface all"),
r"^((?:ethernet|ae)\S+)\s+\d+\s+ukn\/ukn\/down\S+\s+(\S+)\s*$"
):
broken_link = {
"uuid": uuid.uuid1(),
"device_name": device_data.get("device_name"),
"device_type": device_data.get("device_type"),
"interface_name": interface_name,
"interface_name_encoded": interface_name.replace("/", "_"),
"interface_ip_address": "N/A",
"interface_mac_address": interface_mac_address,
}
# Search for latest device_info structure that has broken BGP session in UP state and record its timestamp
device_data_list = db.get_device_data_list__c(
device_data.get("device_name", ""),
command_list=["show interface all"])
for device_data in device_data_list:
regex_interface_name = interface_name.replace("/",
"\\/").replace(
".", "\.")
if netcat.find_regex_ml(
netcat.get_command_output(device_data,
"show interface all"),
rf"(^{regex_interface_name}\s+\S+\s+\S+\/up\s+\S+\s*$)",
hint=interface_name,
optional=False):
broken_link["snapshot_timestamp"] = device_data.get(
"snapshot_timestamp")
break
broken_links.append(broken_link)
else:
netcat.LOGGER.warning(
f"{netcat.fn()}: Unknown device data type value '{device_data.get('type')}'"
)
# Time process execution
end_time = time.monotonic()
netcat.LOGGER.debug(
f"Created broken links list of {len(broken_links)} links in {end_time - start_time:.2f}s"
)
return broken_links
def find_broken_bgp_sessions_per_device(
device_data: Dict[str, Any]) -> List[Dict[str, Any]]:
""" Search for broken BGP sessions on Cisco routers and Palo Alto firewalls """
# Time process execution
start_time = time.monotonic()
# Setup logger to show process name
if os.getpid() != netcat.MAIN_PROCESS_PID:
netcat.bind_logger("SUB_PROC")
broken_bgp_sessions = []
if device_data.get("device_type") == "cisco_router":
# Look for any broken BGP sessions
for bgp_session_peer_ip, bgp_session_peer_asn in netcat.find_regex_ml(
netcat.get_command_output(device_data, "show ip bgp summary"),
r"^(\S+)\s+\d\s+(\d+)\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+\S+\s+(?:Idle|Active)$"
):
broken_bgp_session = {
"uuid": uuid.uuid1(),
"device_name": device_data.get("device_name"),
"device_type": device_data.get("device_type"),
"peer_ip": bgp_session_peer_ip,
"peer_asn": bgp_session_peer_asn,
}
# Search for latest device_data document that has broken BGP session in UP state and record its timestamp
device_data_list = db.get_device_data_list__c(
device_data.get("device_name", ""),
command_list=["show ip bgp summary"])
for device_data in device_data_list:
if netcat.find_regex_ml(
netcat.get_command_output(device_data,
"show ip bgp summary"),
rf"(^{bgp_session_peer_ip}\s+\d\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+\S+\s+\d+$)",
hint=bgp_session_peer_ip,
optional=False):
broken_bgp_session["snapshot_timestamp"] = device_data.get(
"snapshot_timestamp")
break
broken_bgp_sessions.append(broken_bgp_session)
elif device_data.get("device_type") == "paloalto":
# Skip device if its not in active ha state
if netcat.find_regex_sl(
netcat.get_command_output(device_data,
"show high-availability all"),
r"\s+State: (\S+) .*$") not in {
"active", "active-primary", "active-secondary", ""
}:
return []
# Look for any broken BGP sessions
for bgp_session_peer_asn, bgp_session_peer_ip in netcat.find_regex_ml(
netcat.get_command_output(device_data,
"show routing protocol bgp summary"),
r"^\s+peer \S+\s+ AS (\d+), (?:Connect|Active), IP (\S+)$"):
broken_bgp_session = {
"uuid": uuid.uuid1(),
"device_name": device_data.get("device_name"),
"device_type": device_data.get("device_type"),
"peer_ip": bgp_session_peer_ip,
"peer_asn": bgp_session_peer_asn,
}
# Search for latest device_data structure that has broken BGP session in UP state and record its timestamp
device_data_list = db.get_device_data_list__c(
device_data.get("device_name", ""),
command_list=[
"show routing protocol bgp summary",
"show high-availability all"
])
for device_data in device_data_list:
if netcat.find_regex_sl(
netcat.get_command_output(
device_data, "show routing protocol bgp summary"),
rf"(^\s+peer \S+\s+ AS \d+, Established, IP {bgp_session_peer_ip})$",
hint=bgp_session_peer_ip,
optional=False):
broken_bgp_session["snapshot_timestamp"] = device_data.get(
"snapshot_timestamp")
break
broken_bgp_sessions.append(broken_bgp_session)
else:
netcat.LOGGER.warning(
f"{netcat.fn()}: Unknown device data type value '{device_data.get('type')}'"
)
# Time process execution
end_time = time.monotonic()
netcat.LOGGER.debug(
f"Created broken bgp session list of {len(broken_bgp_sessions)} sessions in {end_time - start_time:.2f}s"
)
return broken_bgp_sessions