def download_software(self, requested_software_version: str) -> None:
""" Download software """
major = requested_software_version.split(".")[0]
minor = requested_software_version.split(".")[1]
patch = requested_software_version.split(".")[2]
requested_software_version_dependencies = [f"{major}.0.0"]
if minor != "0":
requested_software_version_dependencies.append(
f"{major}.{minor}.0")
if patch != "0":
requested_software_version_dependencies.append(
f"{major}.{minor}.{patch}")
netcat.LOGGER.info(
"Detected software version dependencies: {}",
" -> ".join(requested_software_version_dependencies))
# Download latest software list
netcat.LOGGER.info("Refreshing available software versions")
available_software_versions = self.send_command(
"request system software check", timeout=120)
if server_error := netcat.find_regex_sl(available_software_versions,
r"(Server error)"):
raise netcat.CustomException(f"Received: '{server_error}'")
def __init__(self, device_info: Dict[str, str]) -> None:
super().__init__(device_info)
if self.type == "cisco_nexus":
self.cli_prompt = rf"{self.name.upper()}(\(conf.*\))?# "
self.password_prompt = "[Pp]assword: "
self.output_formats = OUTPUT_FORMATS_CISCO_NEXUS
elif self.type == "cisco_router":
self.cli_prompt = rf"{self.name.upper()}(\(conf.*\))?#"
self.password_prompt = "Password: "******"cisco_switch":
self.cli_prompt = rf"{self.name.upper()}(\(conf.*\))?#"
self.password_prompt = "[Pp]assword: "
self.output_formats = OUTPUT_FORMATS_CISCO_SWITCH
elif self.type == "cisco_asa":
self.cli_prompt = rf"{self.name.upper()}(\(config\))?# "
self.password_prompt = "password: "******"cisco_asa_mc":
self.cli_prompt = rf"VF(1|2)FW1\/(pri|sec)\/act\/?[A-Z]*(\(config\))?# "
self.password_prompt = "password: "******"Unknown device type: {self.type}")
def send_commit_command(self, timeout: int = 300) -> str:
""" Send commit command to device and wait for it to execute, then return command output """
netcat.LOGGER.info("Configuration commit started")
self.cli.sendline("commit") # type: ignore
expect_output_index = self.cli.expect([
self.cli_prompt,
f"Please synchronize the peers by running 'request high-availability sync-to-remote running-config' first\.\r\n"
+ f"Would you like to proceed with commit\? \(y or n\)"
],
timeout=timeout) # type: ignore
if expect_output_index == 0:
return str(self.cli.before) # type: ignore
if expect_output_index == 1:
netcat.LOGGER.warning(
"Need to synchronise configuration to the other node")
self.send_command("n")
self.exit_config_mode()
self.send_command(
"request high-availability sync-to-remote running-config")
time.sleep(120)
netcat.LOGGER.info("Restarting commit")
return self.commit_config()
raise netcat.CustomException(
f"Problem with expect when executing configuration commit, expect_output_index = {expect_output_index} is out of range"
)
def validate_ha_state(self, snippet: str, timeout: int = 30) -> Any:
""" Validate if HA state of the device is as expected in snipped file """
netcat.LOGGER.info(
"Reading expected HA state from iconfiguration snippet")
# Read expected HA status from snippet
for line in snippet.split("\n"):
if line.find(r"# Expected HA state: ") >= 0:
expected_ha_state = line[20:].strip().lower()
break
else:
netcat.LOGGER.info(
"Configuration snippet doesn't contain information about expected HA state, assuming 'active' state"
)
expected_ha_state = "active"
netcat.LOGGER.info(f"Expected HA state: '{expected_ha_state}'")
netcat.LOGGER.info("Validating device's HA state")
self.cli.sendline("") # type: ignore
expect_output_index = self.cli.expect( # type: ignore
[
rf"{self.username}@{self.name.upper()}\(?({expected_ha_state})\)?[#>] ",
self.cli_prompt
],
timeout=timeout)
if expect_output_index == 0:
return self.cli.before # type: ignore
if expect_output_index == 1:
raise netcat.CustomException(
f"HA state in cli prompt '{self.cli.after}' is not as expected"
) # type: ignore
raise netcat.CustomException(
f"Problem with expect when sending command, expect_output_index = {expect_output_index} is out of range"
)
def wrapper(*args: Any, **kwargs: Any) -> Any:
try:
return function(*args, **kwargs)
except pexpect.EOF as exception:
error_message = exception.value.split("\n")[5]
if "Host key verification failed" in error_message:
raise netcat.CustomException(
"Device identity verification failed")
if "Authorization failed" in error_message:
raise netcat.CustomException("Authorization failed")
if "Connection timed out" in error_message:
raise netcat.CustomException("Connection timeout")
raise netcat.CustomException(f"Expect error '{error_message}'")
except pexpect.TIMEOUT:
raise netcat.CustomException("Connection timeout")
def write(table_name, document, max_attempts=15, max_sleep_time=10):
""" Write document into database table """
from time import sleep
from random import uniform
if table_name in {netcat.DBT_INFO, netcat.DBT_BACKUP}:
document = netcat.compress_device_data(document)
attempts = max_attempts
while attempts:
try:
DB_CLIENT.put_item(
TableName=table_name,
Item=_unfold(document),
)
break
except botocore.exceptions.ClientError as exception:
if exception.response["Error"]["Code"] not in {
"ProvisionedThroughputExceededException",
"ThrottlingException"
}:
raise netcat.CustomException(
f"Botocore: Unable to write document into '{table_name}' table, Botocore exception: '{exception}'"
)
sleep_time = uniform(0.1, max_sleep_time)
attempts -= 1
netcat.LOGGER.warning(
f"Botocore: Unable to write document into '{table_name}', attempt {max_attempts - attempts}, will retry in {sleep_time:.2f} seconds"
)
sleep(sleep_time)
else:
raise netcat.CustomException(
f"Botocore: Unable to write device data document into database table '{table_name}' after {max_attempts} attempts"
)
def load_latest_backup(device_name: str) -> Dict[str, Any]:
""" Returns latest config backup for given device name """
try:
with pymongo.MongoClient(DB_URI) as client:
return netcat.decompress_device_data(
client.get_default_database()[netcat.DBT_BACKUP].find_one(
filter={"device_name": device_name},
sort=[("snapshot_timestamp", pymongo.DESCENDING)],
projection={"_id": False}) or {})
except pymongo.errors.PyMongoError as exception:
raise netcat.CustomException(
f"Unable to load latest config backup, PyMongo exception: '{exception}'"
)
def get_device_data(device_info: Dict[str, str]) -> Dict[str, Dict[str, str]]:
""" Access device and retrieve its current info """
if device_info["device_type"].startswith("paloalto"):
with PACliAccess(device_info) as cli:
return cli.get_device_data()
if device_info["device_type"].startswith("cisco"):
with CiscoCliAccess(device_info) as cli:
return cli.get_device_data()
if device_info["device_type"].startswith("f5"):
with F5CliAccess(device_info) as cli:
return cli.get_device_data()
raise netcat.CustomException(f"Unsupported device type '{device_info['device_type']}'")
def clear_commit_in_progress(self) -> None:
""" Check if there is any commit in progress and wait till finishes or time out after 3 minutes"""
netcat.LOGGER.info("Checking for any other commit in progress")
for _ in range(6):
if netcat.find_regex_sl(
self.send_command("show jobs processed"),
r"(^[^ ]+ [^ ]+ +[^ ]+ +\d+ +Commit +ACT .*$)"):
netcat.LOGGER.warning(
"Another commit in progress, will wait 30s and recheck")
time.sleep(30)
continue
break
else:
raise netcat.CustomException(
"Another commit in progress takes over 3 minutes")
netcat.LOGGER.info("No other commit in progress")
def deploy_config_snippet(device_info: Dict[str, str], snippet: str, site_id_check: bool, inet_gw_check: bool, no_commit: bool) -> None:
""" Access device, read all neccessary local settings and deploy snippet """
def _(cli: Any, snippet: str, site_id_check: bool, inet_gw_check: bool, no_commit: bool = False) -> None:
site_name = ""
site_id = ""
inet_gw = ""
if site_id_check:
site_id = cli.get_site_id()
if inet_gw_check:
inet_gw = cli.get_inet_gw()
if device_info["device_type"] == "paloalto":
site_name = device_info["device_name"][0:-3].upper()
snippet = snippet.format(site_name=site_name, site_id=site_id, inet_gw=inet_gw)
cli.create_config_snapshot()
cli.deploy_config_snippet(snippet, no_commit)
if device_info["device_type"] == "paloalto":
with PACliAccess(device_info) as cli:
_(cli, snippet, site_id_check, inet_gw_check, no_commit)
elif device_info["device_type"] == "cisco_router":
with CiscoCliAccess(device_info) as cli:
_(cli, snippet, site_id_check, inet_gw_check)
elif device_info["device_type"] == "cisco_switch":
with CiscoCliAccess(device_info) as cli:
_(cli, snippet, site_id_check, inet_gw_check)
else:
raise netcat.CustomException(f"Unsupported device type '{device_info['type']}'")
class CiscoCliAccess(netcat_cli.NetCatCliAccess):
""" CLI access class for Cisco devices """
def __init__(self, device_info: Dict[str, str]) -> None:
super().__init__(device_info)
if self.type == "cisco_nexus":
self.cli_prompt = rf"{self.name.upper()}(\(conf.*\))?# "
self.password_prompt = "[Pp]assword: "
self.output_formats = OUTPUT_FORMATS_CISCO_NEXUS
elif self.type == "cisco_router":
self.cli_prompt = rf"{self.name.upper()}(\(conf.*\))?#"
self.password_prompt = "Password: "******"cisco_switch":
self.cli_prompt = rf"{self.name.upper()}(\(conf.*\))?#"
self.password_prompt = "[Pp]assword: "
self.output_formats = OUTPUT_FORMATS_CISCO_SWITCH
elif self.type == "cisco_asa":
self.cli_prompt = rf"{self.name.upper()}(\(config\))?# "
self.password_prompt = "password: "******"cisco_asa_mc":
self.cli_prompt = rf"VF(1|2)FW1\/(pri|sec)\/act\/?[A-Z]*(\(config\))?# "
self.password_prompt = "password: "******"Unknown device type: {self.type}")
def setup_cli(self) -> None:
""" Setup CLI to make it usable for automated operation """
netcat.LOGGER.info("Configuring initial cli setup")
self.clear_pexpect_buffer()
if self.type in {"cisco_asa", "cisco_asa_mc"}:
self.send_command("terminal pager 0")
else:
self.send_command("terminal length 0")
self.send_command("terminal width 500")
@netcat_cli.exception_handler
def get_site_id(self) -> str:
""" Detect site ID """
netcat.LOGGER.info("Detecting Site ID")
# We do it for routers only, makes little sense to do for other devices
if self.type == "cisco_router":
if site_id := netcat.find_regex_sl(
self.send_command("show ip bgp summary"),
r"^BGP router identifier \d+\.(\d+).\d+.\d+,.*$"):
netcat.LOGGER.info(f"Detected Site ID: {site_id}")
return site_id
raise netcat.CustomException(
"Cannot site id in 'show ip bgp summary' comman output")
netcat.LOGGER.info("Detecting Internet default gateway IP address")
# We do it for routers only, makes little sense to do for other devices
if self.type == "cisco_router":
if inet_gw := netcat.find_regex_sl(
self.send_command(
"show running-config | include 0.0.0.0 0.0.0.0"),
r"^ip route (?:vrf INTERNET )?0\.0\.0\.0 0\.0\.0\.0 (\d+\.\d+\.\d+\.\d+) .*$"
):
netcat.LOGGER.info(
"Detected Internet default gateway IP address: {}",
inet_gw)
return inet_gw
raise netcat.CustomException("Cannot detect Site ID")
def enter_config_mode(self) -> None:
""" Enter Cisco configuration mode """
netcat.LOGGER.debug("Entering configuration mode")
self.send_command("configure terminal")
self.clear_pexpect_buffer()
def exit_config_mode(self) -> None:
""" Exit Cisco configuration mode """
netcat.LOGGER.debug("Exiting configuration mode")
self.send_command("end")
self.clear_pexpect_buffer()
class PACliAccess(netcat_cli.NetCatCliAccess):
""" CLI access class for PA devices """
def __init__(self, device_info: Dict[str, str]) -> None:
super().__init__(device_info)
self.cli_prompt = rf"{self.username}@{self.name.upper()}\(?(active-primary|active-secondary|active|passive|non-functional|suspended|)\)?[#>] "
self.password_prompt = r"Password: "******""" Validate if HA state of the device is as expected in snipped file """
netcat.LOGGER.info(
"Reading expected HA state from iconfiguration snippet")
# Read expected HA status from snippet
for line in snippet.split("\n"):
if line.find(r"# Expected HA state: ") >= 0:
expected_ha_state = line[20:].strip().lower()
break
else:
netcat.LOGGER.info(
"Configuration snippet doesn't contain information about expected HA state, assuming 'active' state"
)
expected_ha_state = "active"
netcat.LOGGER.info(f"Expected HA state: '{expected_ha_state}'")
netcat.LOGGER.info("Validating device's HA state")
self.cli.sendline("") # type: ignore
expect_output_index = self.cli.expect( # type: ignore
[
rf"{self.username}@{self.name.upper()}\(?({expected_ha_state})\)?[#>] ",
self.cli_prompt
],
timeout=timeout)
if expect_output_index == 0:
return self.cli.before # type: ignore
if expect_output_index == 1:
raise netcat.CustomException(
f"HA state in cli prompt '{self.cli.after}' is not as expected"
) # type: ignore
raise netcat.CustomException(
f"Problem with expect when sending command, expect_output_index = {expect_output_index} is out of range"
)
def setup_cli(self) -> None:
""" Setup CLI to make it usable for automated operation """
netcat.LOGGER.info("Configuring initial cli setup")
self.send_command("set cli scripting-mode on")
self.clear_pexpect_buffer()
self.send_command("set cli terminal width 500")
self.send_command("set cli terminal height 500")
self.send_command("set cli pager off")
self.send_command("set cli confirmation-prompt off")
@netcat_cli.exception_handler
def clear_commit_in_progress(self) -> None:
""" Check if there is any commit in progress and wait till finishes or time out after 3 minutes"""
netcat.LOGGER.info("Checking for any other commit in progress")
for _ in range(6):
if netcat.find_regex_sl(
self.send_command("show jobs processed"),
r"(^[^ ]+ [^ ]+ +[^ ]+ +\d+ +Commit +ACT .*$)"):
netcat.LOGGER.warning(
"Another commit in progress, will wait 30s and recheck")
time.sleep(30)
continue
break
else:
raise netcat.CustomException(
"Another commit in progress takes over 3 minutes")
netcat.LOGGER.info("No other commit in progress")
@netcat_cli.exception_handler
def get_site_id(self) -> str:
""" Detect site ID """
if site_id := netcat.find_regex_sl(
self.send_command("show routing protocol bgp summary"),
r"^ +router id: +\d+\.(\d+)\.\d+\.\d+$"):
netcat.LOGGER.info(f"Detected Site ID: {site_id}")
return site_id
raise netcat.CustomException(
"Cannot detect site id in 'show routing protocol bgp summary' command output"
)
def deploy_config_snippet(self,
snippet: str,
no_commit: bool = False) -> None:
""" Deploy config line by line i and commit """
# Validate device's HA state
self.validate_ha_state(snippet)
# Wait for another commit to finish, if any
self.clear_commit_in_progress()
# Deploy configuration
netcat.LOGGER.info("Configuration deployment started")
snippet_lines = snippet.split("\n")
self.enter_config_mode()
for line in snippet_lines:
if line and line[0].lstrip() != "#":
netcat.LOGGER.opt(ansi=True).info(
"Deploying line '<cyan>{}</cyan>'", line)
self.send_command(line)
netcat.LOGGER.info("Configuration deployment finished")
self.exit_config_mode()
# Exit if configuration is not supposed to be commited
if no_commit:
netcat.LOGGER.warning(
"Configuration loaded but not commited (per user request)")
return
# Wait for another commit to finish, if any
self.clear_commit_in_progress()
# Commit configuration
self.enter_config_mode()
command_output = self.send_commit_command()
commit_output = command_output.split("\n")[3:-2]
# Check output for commit validation error and report on commit results
commit_validation_error = False
for line in commit_output:
if line.lower().find("error") != -1:
commit_validation_error = True
for line in commit_output:
netcat.LOGGER.opt(ansi=True).info(
"Commit output: <magenta>{}</magenta>", line)
netcat.LOGGER.info("Configuration commit finished")
if commit_validation_error:
self.send_command("revert config")
raise netcat.CustomException(
"Commit validation error detected, reverted to previous configuration"
)
self.exit_config_mode()
netcat.LOGGER.opt(
ansi=True).info("<green>Commit validation successful</green>")
def open_cli(self) -> None:
""" Establish SSH connection to device """
netcat.LOGGER.info("Opening ssh connection to '{}@{}'", self.username,
self.name)
# Check if we need to perform password authentication
if self.auth == "password":
netcat.LOGGER.info("Using password to authenticate")
self.cli = pexpect.spawn(
f"ssh -o PubkeyAuthentication=no -l {self.username} {self.name}",
timeout=60,
encoding="utf-8")
expect_output_index = self.cli.expect([
self.password_prompt,
"Are you sure you want to continue connecting (yes/no)?",
"Connection refused"
])
# Handle situation when connection to device is established for the first time and device's signature needs to be saved
if expect_output_index == 1:
netcat.LOGGER.warning(
"Devices's authenticity record doesn't exist in ssh 'known_hosts' file, adding"
)
self.cli.sendline("yes") # type: ignore
self.cli.expect(self.password_prompt) # type: ignore
# Handle situation when connection to device is being refused, ex. VTY ACL doesn't allow it
if expect_output_index == 2:
raise netcat.CustomException("Connection refused by device")
# Handle situation when first password is rejected (eg. due to device's RADIUS malfunction) and second try is needed
# No more than two tries are attempted to do not lock user out on device
for second_password_attempt in [False, True]:
self.cli.sendline(self.password) # type: ignore
netcat.LOGGER.info("Password sent, waiting for cli prompt")
expect_output_index = self.cli.expect(
[self.cli_prompt, self.password_prompt]) # type: ignore
# Cli pompt received
if expect_output_index == 0:
netcat.LOGGER.info("Cli prompt received")
return
# Wrong username / password on first try, will retry as that might been just a Radius glitch
if expect_output_index == 1 and not second_password_attempt:
netcat.LOGGER.warning(
"Unsuccessful login with supplied credetnials, retrying after 5s"
)
time.sleep(5)
continue
# Wrong username / password on second try, aborting
if expect_output_index == 1 and second_password_attempt:
raise netcat.CustomException(
"Cannot login with supplied credentials")
raise netcat.CustomException(
f"Problem with expect when logging to device, expect_output_index = {expect_output_index} is out of range"
)
# Check if we need to perform RSA authentication
elif self.auth == "rsa":
netcat.LOGGER.info("Using RSA key to authenticate")
self.cli = pexpect.spawn(
f"ssh -o PubkeyAuthentication=yes -l {self.username} {self.name}",
timeout=60,
encoding="utf-8")
# Prevent hypotetical lockdown due to 'infinite authenticity record missing' event
for known_hosts_check_happened_already in [False, True]:
expect_output_index = self.cli.expect( # type: ignore
[
self.cli_prompt,
"Are you sure you want to continue connecting (yes/no)?",
self.password_prompt, "Connection refused"
])
# Cli prompt received
if expect_output_index == 0:
netcat.LOGGER.info("Cli prompt received")
return
# Handle situation when connection to device is established for the first time and device's signature needs to be saved
if expect_output_index == 1 and not known_hosts_check_happened_already:
netcat.LOGGER.warning(
"Device's authenticity record doesn't exist in ssh 'known_hosts' file, adding"
)
self.cli.sendline("yes") # type: ignore
continue
# Handle situation when password prompt is presented
if expect_output_index == 2:
raise netcat.CustomException(
"RSA authentication error, key may not be valid")
# Handle situation when connection to device is being refused, ex. VTY ACL doesn't allow it
if expect_output_index == 3:
raise netcat.CustomException(
"Connection refused by device")
raise netcat.CustomException(
f"Problem with expect when logging to device, expect_output_index = {expect_output_index} is out of range"
)
else:
raise netcat.CustomException(
f"Unknown authentication type '{self.auth}'")
def upgrade_software(self, requested_software_version: str) -> None:
""" Upgrade software """
# Make up to three attempts to install software
for _ in range(3):
for _ in range(30):
command_output = self.send_command(
f"request system software install version {requested_software_version}"
)
if netcat.find_regex_sl(command_output, r"(Server error)"):
if netcat.find_regex_sl(command_output,
r"(install is in progress)"):
netcat.LOGGER.info(
"Another installation in progress, waiting...")
time.sleep(10)
continue
if netcat.find_regex_sl(
command_output,
r"(pending jobs in the commit task queue)"):
netcat.LOGGER.info(
"Pending jobs in commit task queue, waiting...")
time.sleep(10)
continue
if netcat.find_regex_sl(command_output,
r"(commit is in progress)"):
netcat.LOGGER.info("Commit is in progress, waiting...")
time.sleep(10)
continue
raise netcat.CustomException(
f"Received: '{netcat.find_regex_sl(command_output, r'(Server error)')}'"
)
break
else:
raise netcat.CustomException(
"Another installation in progress for over 5 minutes")
job_id = netcat.find_regex_sl(
command_output,
r"^Software install job enqueued with jobid (\d+)\.\s+.*$")
netcat.LOGGER.info(
f"Installation of software version {requested_software_version} started with job id '{job_id}'"
)
time.sleep(5)
while ((
command_output :=
self.send_command(f"show jobs id {job_id}")
) and netcat.find_regex_sl(
command_output,
rf"^\d\S+\s+\S+\s+(?:\S+\s+)?\d+\s+SWInstall\s+(\S+)\s+\S+\s+\S+\s*$"
) in {"ACT", "QUEUED"}):
installation_progress = netcat.find_regex_sl(
command_output,
rf"^\d\S+\s+\S+\s+\S+\s+\d+\s+SWInstall\s+\S+\s+\S+\s+(\S+)\s*$"
)
netcat.LOGGER.info(
f"Installing software version {requested_software_version}, progress {installation_progress}"
)
time.sleep(5)
if netcat.find_regex_sl(
command_output,
rf"^\d\S+\s+\S+\s+\S+\s+\d+\s+SWInstall\s+FIN\s+(\S+)\s+\S+\s*$"
) == "OK":
netcat.LOGGER.info(
f"Installation of software version {requested_software_version} completed"
)
break
netcat.LOGGER.warning(
f"Installation of version {requested_software_version} failed, will retry up to three times..."
)
print("***", command_output, "***")
@netcat_cli.exception_handler
def get_inet_gw(self) -> str:
""" Detect Internet default gateway """
netcat.LOGGER.info("Detecting Internet default gateway IP address")
if inet_gw := netcat.find_regex_sl(
self.send_config_command(
"show network virtual-router VR_GLOBAL routing-table ip static-route SR_DEFAULT nexthop"
), r"^.+ (\d+\.\d+\.\d+.\d+)$"):
netcat.LOGGER.info(
f"Detected Internet default IP address: {inet_gw}")
return inet_gw
raise netcat.CustomException(
"Cannot find 'desitnation' == '0.0.0.0/0 in 'show routing route type static virtual-router VR_GLOBAL' command output"
)
def enter_config_mode(self) -> None:
""" Enter PA configuration mode """
netcat.LOGGER.debug("Entering configuration mode")
self.send_command("configure")
def exit_config_mode(self) -> None:
""" Exit PA configuration mode """
netcat.LOGGER.debug("Exiting configuration mode")
self.send_command("exit")
@netcat_cli.exception_handler