def test_startshell_context(self, mock_close, mock_open):
with StartShell(self.dev) as shell:
shell._chan = MagicMock()
shell.send('test')
mock_close.assert_called_once_with()
def _issu_requirement_validation(self):
"""
Checks:
* The master Routing Engine and backup Routing Engine must be
running the same software version before you can perform a
unified ISSU.
* Check GRES is enabled
* Check NSR is enabled
* Check commit synchronize is enabled
* Verify that NSR is configured on the master Routing Engine
by using the "show task replication" command.
* Verify that GRES is enabled on the backup Routing Engine
by using the show system switchover command.
:returns:
* ``True`` if validation passes.
* * ``False`` otherwise
"""
self.log('ISSU requirement validation: The master Routing Engine and\n'
'backup Routing engine must be running the same software\n'
'version before you can perform a unified ISSU.')
if not (self._dev.facts['2RE'] and self._dev.facts['version_RE0']
== self._dev.facts['version_RE1']):
self.log('Requirement FAILED: The master Routing Engine (%s) and\n'
'backup Routing Engine (%s) must be running the same\n'
'software version before it can perform a unified ISSU' %
(self._dev.facts['version_RE0'],
self._dev.facts['version_RE1']))
return False
if not self._issu_nssu_requirement_validation():
return False
self.log('Verify that GRES is enabled on the backup Routing Engine\n'
'by using the command "show system switchover"')
output = ''
try:
op = self._dev.rpc.request_shell_execute(
routing_engine='backup', command="cli show system switchover")
if op.findtext('.//switchover-state', default='').lower() == 'on':
self.log('Graceful switchover status is On')
return True
output = op.findtext('.//output', default='')
except RpcError:
# request-shell-execute rpc is not available for <14.1
with StartShell(self._dev) as ss:
ss.run('cli', '> ', timeout=5)
if ss.run('request routing-engine '
'login other-routing-engine')[0]:
# depending on user permission, prompt will go to either
# cli or shell, below line of code prompt will finally end
# up in cli mode
ss.run('cli', '> ', timeout=5)
data = ss.run('show system switchover', '> ', timeout=5)
output = data[1]
ss.run('exit')
else:
self.log('Requirement FAILED: Not able run '
'"show system switchover"')
return False
gres_status = re.search(r'Graceful switchover: (\w+)', output, re.I)
if not (gres_status is not None
and gres_status.group(1).lower() == 'on'):
self.log('Requirement FAILED: Graceful switchover status '
'is not On')
return False
self.log('Graceful switchover status is On')
return True
def setUp(self, mock_connect):
self.dev = Device(host='1.1.1.1')
self.shell = StartShell(self.dev)
'''This script is to execute the shell level commands in an infinite loop. How to run the script# from a Ubuntu server running python3
$ python3 --version
Python 3.4.3
$python3 <.pyfilename>, the script makes use of PYEZ juniper library'''
#!/usr/bin/env python3
'''This script is to execute the shell level commands'''
import time
from jnpr.junos.utils.start_shell import StartShell
from jnpr.junos import Device
dev = Device(host='x.x.x.x', user='******', password='******')
dev.open()
with StartShell(dev) as shell:
i = 0
while i < 1:
threads = shell.run('cprod -A fpc0 -c "show threads cpu"')
print("at {} threads command".format(time.asctime()))
for lines in threads:
print(lines)
time.sleep(2)
dev.close()
def get(self, *vargs, **kvargs):
"""
Retrieve the XML (string blob under <output> tag of table data from the
Device instance and returns back the Table instance - for call-chaining
purposes.
If the Table was created with a :path: rather than a Device,
then this method will load the string blob from that file. In this
case, the \*vargs, and \**kvargs are not used.
ALIAS: __call__
:vargs:
[0] is the table :arg_key: value. This is used so that
the caller can retrieve just one item from the table without
having to know the Junos RPC argument.
:kvargs:
these are the name/value pairs relating to the specific Junos
command attached to the table. For example, if the command
is 'show ithrottle id {{ id }}', there is a parameter 'id'.
Any valid command argument can be passed to :kvargs: to further filter
the results of the :get(): operation. neato!
NOTES:
If you need to create a 'stub' for unit-testing
purposes, you want to create a subclass of your table and
overload this methods.
"""
self._clearkeys()
if self._path and self.data:
raise AttributeError("path and data are mutually exclusive")
if self._path is not None:
# for loading from local file-path
with open(self._path, "r") as fp:
self.data = fp.read().strip()
if self.data.startswith("<output>") and self.data.endswith(
"</output>"):
self.data = etree.fromstring(self.data).text
if "target" in kvargs:
self.TARGET = kvargs["target"]
if "key" in kvargs:
self.KEY = kvargs["key"]
if "key_items" in kvargs:
self.KEY_ITEMS = kvargs["key_items"]
if "filters" in kvargs:
self.VIEW.FILTERS = ([kvargs["filters"]] if isinstance(
kvargs["filters"], str) else kvargs["filters"])
cmd_args = self.CMD_ARGS.copy()
if "args" in kvargs and isinstance(kvargs["args"], dict):
cmd_args.update(kvargs["args"])
if len(cmd_args) > 0:
self.GET_CMD = Template(self.GET_CMD).render(**cmd_args)
if self.data is None:
# execute the Junos RPC to retrieve the table
if hasattr(self, "TARGET"):
if self.TARGET is None:
raise ValueError('"target" value not provided')
rpc_args = {
"target": self.TARGET,
"command": self.GET_CMD,
"timeout": "0",
}
try:
self.xml = getattr(self.RPC,
"request_pfe_execute")(**rpc_args)
self.data = self.xml.text
ver_info = self._dev.facts.get("version_info")
if ver_info and ver_info.major[0] <= 15:
# Junos <=15.x output has output something like below
#
# <rpc-reply>
# <output>
# SENT: Ukern command: show memory
# GOT:
# GOT: ID Base Total(b) Free(b) Used(b)
# GOT: -- -------- --------- --------- ---------
# GOT: 0 44e72078 1882774284 1689527364 193246920
# GOT: 1 b51ffb88 67108860 57651900 9456960
# GOT: 2 bcdfffe0 52428784 52428784 0
# GOT: 3 b91ffb88 62914556 62914556 0
# LOCAL: End of file
# </output>
# </rpc-reply>
# hence need to do cleanup
self.data = self.data.replace("GOT: ", "")
except RpcError:
with StartShell(self.D) as ss:
ret = ss.run('cprod -A %s -c "%s"' %
(self.TARGET, self.GET_CMD))
if ret[0]:
self.data = ret[1]
else:
self.xml = self.RPC.cli(self.GET_CMD)
self.data = self.xml.text
if self.USE_TEXTFSM:
self.output = self._parse_textfsm(platform=self.PLATFORM,
command=self.GET_CMD,
raw=self.data)
else:
# state machine
sm = StateMachine(self)
self.output = sm.parse(self.data.splitlines())
# returning self for call-chaining purposes, yo!
return self
def get_max_connections(self):
"""Use shell commands to find maximum allowed connection-limit."""
# the list of commands that will:
# - exit from the command shell to the Junos CLI
# - enter configuration mode
# - issue the command "set system services ssh connection-limit ?",
# which will return help information we want to process
# - exit configuration mode
shell_commands = [
{'command': 'exit', 'prompt': '> ', 'max': False},
{'command': 'configure', 'prompt': '# ', 'max': False},
{'command': 'set system services ssh connection-limit ?',
'prompt': '# ', 'max': True},
{'command': 'exit', 'prompt': '> ', 'max': False}
]
# open a command shell on the device
shell = StartShell(self.dev)
shell.open()
# iterate over the list of commands, capturing the output from
# the command in whose results we are interested ('max' = True)
max_msg = None
for shellcmd in shell_commands:
shellout = shell.run(shellcmd['command'], shellcmd['prompt'])
self.results['shell_results'].append(shellout)
if shellout[0] is False:
msg = 'Shell command "%s" did not complete as expected: %s' \
% (shellcmd['command'], shellout[1])
raise RuntimeError(msg)
if shellcmd['max']:
max_msg = shellout[1]
shell.close()
# process the command output to find the max allowed value
if max_msg is not None:
max_arr = max_msg.splitlines()
regex = r'connection-limit[^\(\[]*[\(\[]\d+\.\.(\d+)'
max_str = None
for line in max_arr:
m = re.search(regex, line, flags=re.IGNORECASE)
if m is not None:
max_str = m.group(1)
break
if max_str is not None:
reported_max = int(max_str)
self.results['connection_max'] = reported_max
if reported_max < self.desired_connection_limit:
self.results['connection_limit'] = reported_max
else:
self.results['connection_limit'] = \
self.desired_connection_limit
else:
msg = 'Regex match expected but not found in command results'
raise ValueError(msg)
else:
msg = 'Missing expected results from shell commands.'
raise ValueError(msg)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--device", help="target device to monitor")
parser.add_argument("--user", help="username")
parser.add_argument("--password", help="password")
parser.add_argument("--interval",
default=3,
help="interval to run monitoring")
parser.add_argument(
"--holddown",
default=180,
help=
"time that a device must not exist in a converging state before calculating total elapsed convergence time"
)
parser.add_argument("--test_id",
help="Identifier for the test being performed")
args = parser.parse_args()
if args.device and args.user and args.password and args.interval and args.test_id:
hostname = socket.gethostbyname(args.device)
print "###Running on device " + args.device + " as user " + args.user + " with an interval of " + str(
args.interval) + " seconds and minimum holddown of " + str(
args.holddown) + " ###"
iterations = 0
interval = float(args.interval)
###Initialize the dictionary for storing output values gathered from the devices
for i in tables:
stats_dict[i] = {}
stats_dict[i]['raw'] = []
stats_dict[i]['stats'] = {}
with Device(host=hostname,
port=22,
username=args.user,
password=args.password) as dev:
dev_shell = StartShell(dev)
dev_shell.open()
last_mutation_count = 0
initial_convergence_timestamp = None
most_recent_convergence_timestamp = None
while (True):
route_manager = dev_shell.run(
'cprod -A fpc0 -c "show route manager statistics"')
###Capture the current timestamp
timestamp = datetime.datetime.utcnow()
stats = route_manager[1]
if "Permission denied" in stats:
print "Permission to execute cprod command failed"
sys.exit(0)
###Parse the output of show route manager statistics and load into the stats dictionary
parse_route_manager_stats(stats_dict, stats)
###Sort the stats dictionary
load_stats_dict(stats_dict)
###Determine if any mutations have occurred - if so, then a convergence is occurring
log_prefix = str(
timestamp
) + " device: " + args.device + " test_id: " + args.test_id + " "
if last_mutation_count != stats_dict['total_mutations']:
if iterations > 0:
print log_prefix + "Convergence occurring - total FIB mutations since last change: " + str(
stats_dict['total_mutations'] -
last_mutation_count)
most_recent_convergence_timestamp = timestamp
if initial_convergence_timestamp is None:
initial_convergence_timestamp = timestamp
for table in tables:
for key in stats_dict[table]['stats']:
for metric in metrics:
delta_metric = 'delta_' + metric
if metric == key:
if int(stats_dict[table]['stats']
[delta_metric]) > 0:
print " " + str(
timestamp
) + " " + table + " " + metric + " " + str(
stats_dict[table]['stats']
[delta_metric])
last_mutation_count = stats_dict['total_mutations']
stats_dict['total_mutations'] = 0
###No convergence has occurred since mutations are the same since the most recent iteration, determine if holddown has expired
else:
if initial_convergence_timestamp is not None:
time_delta = most_recent_convergence_timestamp + datetime.timedelta(
seconds=int(args.holddown))
if timestamp >= (time_delta):
print log_prefix + " Total time spent in a converging state: " + str(
most_recent_convergence_timestamp -
initial_convergence_timestamp)
initial_convergence_timestamp = None
iterations += 1
sleep(interval)
else:
parser.print_usage()
def _issu_requirement_validation(self):
"""
Checks:
* The master Routing Engine and backup Routing Engine must be
running the same software version before you can perform a
unified ISSU.
* Check GRES is enabled
* Check NSR is enabled
* Check commit synchronize is enabled
* Verify that NSR is configured on the master Routing Engine
by using the "show task replication" command.
* Verify that GRES is enabled on the backup Routing Engine
by using the show system switchover command.
:returns:
* ``True`` if validation passes.
* * ``False`` otherwise
"""
self.log("ISSU requirement validation: The master Routing Engine and\n"
"backup Routing engine must be running the same software\n"
"version before you can perform a unified ISSU.")
if not (self._dev.facts["2RE"] and self._dev.facts["version_RE0"]
== self._dev.facts["version_RE1"]):
self.log("Requirement FAILED: The master Routing Engine (%s) and\n"
"backup Routing Engine (%s) must be running the same\n"
"software version before it can perform a unified ISSU" %
(self._dev.facts["version_RE0"],
self._dev.facts["version_RE1"]))
return False
if not self._issu_nssu_requirement_validation():
return False
self.log("Verify that GRES is enabled on the backup Routing Engine\n"
'by using the command "show system switchover"')
output = ""
try:
op = self._dev.rpc.request_shell_execute(
routing_engine="backup", command="cli show system switchover")
if op.findtext(".//switchover-state", default="").lower() == "on":
self.log("Graceful switchover status is On")
return True
output = op.findtext(".//output", default="")
except RpcError:
# request-shell-execute rpc is not available for <14.1
with StartShell(self._dev) as ss:
ss.run("cli", "> ", timeout=5)
if ss.run("request routing-engine "
"login other-routing-engine")[0]:
# depending on user permission, prompt will go to either
# cli or shell, below line of code prompt will finally end
# up in cli mode
ss.run("cli", "> ", timeout=5)
data = ss.run("show system switchover", "> ", timeout=5)
output = data[1]
ss.run("exit")
else:
self.log("Requirement FAILED: Not able run "
'"show system switchover"')
return False
gres_status = re.search(r"Graceful switchover: (\w+)", output, re.I)
if not (gres_status is not None
and gres_status.group(1).lower() == "on"):
self.log("Requirement FAILED: Graceful switchover status "
"is not On")
return False
self.log("Graceful switchover status is On")
return True
def clear_ospf(self):
ss = StartShell(self.dev)
ss.open()
ss.run('cli -c "clear ospf neighbor area 0"')
from jnpr.junos import Device
from jnpr.junos.utils.start_shell import StartShell
import time
dev = Device(host='crpd01', user='******', passwd='lab123')
ss = StartShell(dev)
with StartShell(dev) as ss:
for x in range(0, 17):
output = ss.run('cli -c "clear bgp neighbor 192.168.60.2"')
print(output)
time.sleep(10)
print("180 seconds have elapsed. END")
def _ssh_exec(self, command):
with StartShell(self._dev) as sh:
got = sh.run(command)
return got
from jnpr.junos import Device
from jnpr.junos.utils.start_shell import StartShell
from getpass import getpass
from pprint import pprint
juniper_srx = {
"host": "srx1.twb-tech.com",
"user": "******",
"password": getpass(),
}
print()
for device in [juniper_srx]:
a_device = Device(**device)
a_device.open()
ss = StartShell(a_device)
ss.open()
ss.run(
'cli -c "request support information | save /var/tmp/information.txt"')
version = ss.run('cli -c "show version"')
print(version)
ss.close()
a_device.close()
print()
'''
from jnpr.junos import Device
from jnpr.junos.utils.start_shell import StartShell
dev = Device(host='router1.example.net')
dev.open()
with Device(host=router_ip, user=dev_username, passwd=dev_password, port='830') as dev:
try:
dev.open()
dev.timeout = 300
except ConnectError as err:
print("Cannot connect to device: {0}".format(err))
sys.exit(1)
except Exception as err:
print(err)
sys.exit(1)
hostname = dev.facts['hostname']
print(hostname)
if (dev.facts['RE0'] == None or dev.facts['RE0']['mastership_state'] == 'Present') and (dev.facts['RE1']['mastership_state']) == 'master':
print ("Only RE1 is Available on this Router")
with StartShell(dev, timeout=100) as ss:
if ss.run('cli', '>')[0]:
ss.run('request support information | save RSI_' + hostname + '_' + current_date + '.txt', '>')
ss.run('file archive compress source /var/log/* destination re1_var_logs' + '_' + hostname + '_' + current_date + '.tgz', '>')
if ss.run('sftp '<sftp-server-username>@<sftp-server-ip>', 'password:'******'sftp <sftp-server-username>@<sftp-server-ip>', '(yes/no)?')[0]:
data0 = ss.run('yes', 'password:'******'<sftp-server-pwd>', '>')
print(data3)
data4 = ss.run('cd <sftp-server-folder>', '>')
print(data4)
data5 = ss.run('put RSI_' + hostname + '_' + current_date + '.txt', '>')
print(data5)
data6 = ss.run('put re1_var_logs_' + hostname + '_' + current_date + '.tgz', '>')
print(data6)
ss.close()
def main():
"""
Generic main() statement
"""
parser = argparse.ArgumentParser()
parser.add_argument('--user',
nargs=1,
help='user to authenticate on remote host')
parser.add_argument('--password',
nargs=1,
help='password to authenticate on remote host')
parser.add_argument('--host', nargs=1, help='remote host to connect to')
parser.add_argument('filepath', help='Path to filename to work on')
args = parser.parse_args()
if not (args.user and args.password and args.host):
print("must specify --host, --user and --password args")
sys.exit(1)
print("script started at {}".format(datetime.datetime.now()))
host = args.host[0]
user = args.user[0]
password = args.password[0]
if re.search('/', args.filepath):
file_name = args.filepath.rsplit('/', 1)[1]
else:
file_name = args.filepath
if args.filepath + ".sha1":
sha1file = open(args.filepath + ".sha1", "r")
orig_sha1 = sha1file.read().rstrip()
else:
sha1_str = call("sha1sum {}".format(args.filepath))
orig_sha1 = sha1_str.split(" ")[0]
file_size = os.path.getsize(args.filepath)
split_size = divmod(file_size, 20)[0]
with tempdir():
call("split -b {} {} {}".format(split_size, args.filepath, file_name),
shell=True)
sfiles = []
for sfile in os.listdir('.'):
if fnmatch.fnmatch(sfile, '{}*'.format(file_name)):
sfiles.append(sfile)
dev = Device(host=host, user=user, passwd=password)
with StartShell(dev) as s_sh:
s_sh.run("rm -rf /var/tmp/splitcopy_{}".format(file_name))
s_sh.run("mkdir /var/tmp/splitcopy_{}".format(file_name))
if not s_sh.last_ok:
print("unable to create the tmp directory -"
" is there sufficient disk space on remote host?,"
" exiting")
sys.exit(1)
loop = asyncio.get_event_loop()
tasks = []
for sfile in sfiles:
task = loop.run_in_executor(
None, functools.partial(scp_put, dev, sfile, file_name))
tasks.append(task)
loop.run_until_complete(asyncio.gather(*tasks))
loop.close()
s_sh.run("cat /var/tmp/splitcopy_{}/* > /var/tmp/{}".format(
file_name, file_name))
s_sh.run("rm -rf /var/tmp/splitcopy_{}".format(file_name))
if not s_sh.last_ok:
print("unable to delete the tmp directory on remote host,"
" delete it manually")
if s_sh.run("ls /var/tmp/{}".format(file_name)):
sha1_tuple = s_sh.run("sha1 /var/tmp/{}".format(file_name))
new_sha1 = (
sha1_tuple[1].split("\n")[1].split(" ")[3].rstrip())
if orig_sha1 == new_sha1:
print("file has been successfully copied to {}:/var/tmp/{}"
", sha1 matches".format(host, file_name))
else:
print("file has been copied to {}:/var/tmp/{}, but the "
"sha1 does not match - please retry".format(
host, file_name))
else:
print("file {}:/var/tmp/{} not found! please retry".format(
host, file_name))
dev.close()
print("script ended at {}".format(datetime.datetime.now()))
def script2(hostname):
######### Prologue stuff
# Remove old log file if one exists for the given hostname
if os.path.exists('./logs/' + hostname):
os.remove('./logs/' + hostname)
# Start a new log file
log(hostname, "------------------------")
# Variables we'll use in different places. Same quick method for assigning username and password in one line
(username,
password) = open('./config/credentials.txt').read().splitlines()[:2]
root_password = open('./config/root_password.txt').readline()
rc_file_location = '/etc/rc.mount.platform'
rc_md5 = 'e4e17e0aa795f4dd790ed3f15f18859b'
# Can't use context managers here since we're rebooting
# Attempt to connect to the given hostname
try:
dev = Device(host=hostname, user=username, passwd=password)
dev.open()
except Exception as e:
log(hostname, 'Failed to connect for Device()')
return {'success': 0, 'message': 'Failed to connect to Device()'}
# Attempt to create a shell connection to the given hostname
try:
shell = StartShell(dev)
shell.open()
except:
log(hostname, 'Failed to connect for StartShell()')
return {'success': 0, 'message': 'Failed to connect to StartShell()'}
# RC file check - We do this using Junos's RPC for "file list" op command, this is exactly
# like sending "file list /etc/rc.mount.platform" from the command-line
file_show = dev.rpc.file_list(path=rc_file_location)
file_show = file_show.xpath('//total-files')
# If the length of file_show variable is 0, that means the file wasn't found
if len(file_show) < 1:
return {
'success':
0,
'message':
'RC file not found in /etc/rc.mount.platform, didn\'t show in ls listing'
}
file_show = str(file_show[0].text).strip()
log(hostname, 'file_show: ' + file_show)
# Excessive check most likely, but I wanted to be certain. Pretty much the same check as the previous check
if file_show != "1":
return {
'success': 0,
'message': 'RC file not found in /etc/rc.mount.platform'
}
############################################
# MD5 file check - We do this using Junos's RPC for "file checksum md5"
md5_rc_check = dev.rpc.get_checksum_information(path=rc_file_location)
md5_rc_check = md5_rc_check.xpath('//checksum')
# If the length of md5_rc_check is less than 0, that means no checksum XML node was found, which means something
# went wrong, or that the file didn't exist. Realistically we probably don't need this here because execution would
# not make it this far due to our previous checks, but I wanted to be explicit about the checks we're doing and
# follow the MOP as exact as I could
if len(md5_rc_check) < 1:
return {
'success': 0,
'message': 'MD5 check on RC file unable to be executed'
}
md5_rc_check = str(md5_rc_check[0].text).strip()
# Log / print information about the step we're on so the user is aware of where the script is currently at in its
# execution
log(hostname,
'md5 of rc file: ' + md5_rc_check + ' compared to: ' + rc_md5)
# If the checksum doesn't match our hard-coded/expected checksum of the file, we report a failure back to master.py
# in the form of a dict
if md5_rc_check != rc_md5:
return {
'success':
0,
'message':
'MD5 check FAILED. RC on device is corrupted, did not match ' +
rc_md5
}
############################################
# Switch StartShell with su root - We need this to run some of the commands later on. StartShell() is an
# "extension" for PyEZ that allows us simple access to shell
shell.run('su root', this='Password:'******'whoami')
if 'whoami\r\r\nroot\r\n' not in whoami[1]:
log(hostname,
'INVALID ROOT PASSWORD GIVEN, COULD NOT SU TO ROOT. EXITING.')
return {
'success': 0,
'message': 'Invalid root password given. Exiting.'
}
# c. Run fsck utility on the Unmounted partition to make sure its clean
# Check which is backup partition (/dev/da0s1a or /dev/da0s2a) using "show system snapshot media internal" RPC
show_system_snapshot = dev.rpc.get_snapshot_information(media='internal')
snapshot_information = show_system_snapshot.xpath('//snapshot-information')
if len(snapshot_information) < 1:
return {
'success':
0,
'message':
'Unable to retrieve "show system snapshot media internal" output'
}
snapshot_information = snapshot_information[0]
snapshot_medium = snapshot_information.xpath('//snapshot-medium')
partitions = {}
for medium in snapshot_medium:
temp = str(medium.text).split('internal (')[1]
if 'primary' in temp:
temp = temp.split(') (primary)')[0]
partitions['primary'] = temp
else:
temp = temp.split(') (backup)')[0]
partitions['backup'] = temp
log(hostname, 'partitions: ' + json.dumps(partitions))
fsck_output = shell.run('fsck -f -y ' + partitions['backup'])
log(hostname, "fsck -f -y " + partitions['backup'] + "\n" + fsck_output[1])
# d. Perform snapshot slice alternate and save rescue configuration
########## Save rescue configuration - request system configuration rescue save
log(hostname, "request configuration rescue save - Starting")
request_rescue_configuration_save = dev.rpc.request_save_rescue_configuration(
dev_timeout=500)
log(hostname, "request configuration rescue save - Completed")
#@TODO: Not sure if there's error-handling we need to do here. There is a //success xpath we can check for
log(hostname, "request system snapshot slice alternate - Starting")
request_system_snapshot_slice_alternate = dev.rpc.request_snapshot(
slice="alternate", dev_timeout=500)
log(hostname, "request system snapshot slice alternate - Completed")
########### REBOOT THE DEVICE
request_system_reboot = dev.rpc.request_reboot()
log(
hostname,
"REBOOTING - Initializing sleep to give device time to actually reboot"
)
# Need to kill the dev and shell connections or it'll error out.
# Need to sleep to give device time to reboot. Junos waits 60 seconds before initiating shutdown process, so
# I pad 30 seconds on to give it time to actually shutdown.
shell.close()
dev.close()
time.sleep(90)
# While loop to reconnect to rebooted device, attemptToConnectToRebootedDevice() returns a Device() instantiation
log(hostname, "Beginning post-reboot reconnect attempts")
reconnDev = attemptToConnectToRebootedDevice(hostname, username, password)
# this means we timed out... device needs manual intervention
if reconnDev == False:
log(
hostname,
"Post-reboot FAILED. Device failed to come back up - Manual intervention required"
)
return {
'success':
0,
'message':
"Post-reboot FAILED. Device failed to come back up - Manual intervention required"
}
log(hostname, "Post-reboot reconnection successful!")
log(
hostname,
"Checking if we booted from backup partition - show system storage partitions"
)
# Check if backup partition was booted
partition_check = reconnDev.rpc.get_system_storage_partitions()
partition_check = partition_check.xpath('//partitions/booted-from')
if len(partition_check) > 1:
return {
'success':
0,
'message':
"Couldn't determine active/backup partition from 'show system storage partitions'"
}
partition_check = str(partition_check[0].text).strip()
log(hostname,
'show system storage partitions: booted from ' + partition_check)
################################################################################
################################################################################
################################################################################
################################################################################
# If we booted from backup, COMMENCE THE "OTHER" SCRIPT (MOP-Clean-Primary-partition.pdf).
# I should probably extract this out into a separate function, but I don't want to deal with figuring out what
# variables I need to pass-thru
if partition_check == "backup":
########### Check which is backup partition (/dev/da0s1a or /dev/da0s2a)
show_system_snapshot = reconnDev.rpc.get_snapshot_information(
media='internal')
snapshot_information = show_system_snapshot.xpath(
'//snapshot-information')
if len(snapshot_information) < 1:
return {
'success':
0,
'message':
'Unable to retrieve "show system snapshot media internal" output'
}
snapshot_information = snapshot_information[0]
snapshot_medium = snapshot_information.xpath('//snapshot-medium')
partitions = {}
for medium in snapshot_medium:
temp = str(medium.text).split('internal (')[1]
if 'primary' in temp:
temp = temp.split(') (primary)')[0]
partitions['primary'] = temp
else:
temp = temp.split(') (backup)')[0]
partitions['backup'] = temp
log(hostname, 'partitions: ' + json.dumps(partitions))
fsck_output = shell.run('fsck -f -y ' + partitions['backup'])
log(hostname,
"fsck -f -y " + partitions['backup'] + "\n" + fsck_output[1])
# 3. Take a snapshot to the alternate partition (this is a hidden command), then compare "show system snapshot media internal" to ensure
# both partitions have the same versions
log(
hostname,
'request system snapshot media internal slice alternate - Starting'
)
request_system_snapshot_media_internal_slice_alternate = reconnDev.rpc.request_snapshot(
slice='alternate', media='internal', dev_timeout=500)
log(
hostname,
'request system snapshot media internal slice alternate - Completed'
)
# 3a. Packages should be the same on both slices
log(hostname, 'show system snapshot media internal - Started')
show_system_snapshot_media_internal = reconnDev.rpc.get_snapshot_information(
media='internal')
show_system_snapshot_media_internal = show_system_snapshot_media_internal.xpath(
'//software-version')
log(
hostname,
'show system snapshot media internal - Comparing packages between slices'
)
software_versions = {}
for software_index, software_version in enumerate(
show_system_snapshot_media_internal):
packages = software_version.xpath('./package')
software_versions.setdefault(software_index, {})
for package_index, package in enumerate(packages):
package_name = str(
package.xpath('./package-name')[0].text).strip()
package_version = str(
package.xpath('./package-version')[0].text).strip()
software_versions[software_index][
package_name] = package_version
packages_match = True
for package_name in software_versions[0].keys():
if software_versions[0][package_name] != software_versions[1][
package_name]:
packages_match = False
break
if packages_match != True:
log(
hostname,
"Packages from 'show system snapshot media internal' do not match"
)
return {
'success':
0,
'message':
"Packages from 'show system snapshot media internal' do not match"
}
log(hostname,
'show system snapshot media internal - Comparison completed')
#######################
# 4. show system storage partitions - The MOP doesn't really explain what we're looking for with this command. Skipping for now.
# 5. show system alarms - Check for alarm-description containing "Boot from backup root", if we find one, run "request system reboot slice alternate media internal"
show_system_alarms = reconnDev.rpc.get_system_alarm_information()
show_system_alarms = show_system_alarms.xpath('//alarm-description')
for alarm_description in show_system_alarms:
alarm_description = str(alarm_description.text).strip()
# If we see the backup root alarm, we need to reboot AGAIN
if "Boot from backup root" in alarm_description:
log(
hostname,
'REBOOTING - Backup root alarm FOUND - request system reboot slice alternate media internal'
)
reconnDev.rpc.request_reboot(media='internal',
slice='alternate')
reconnDev.close()
time.sleep(90)
# While loop to reconnect to rebooted device, attemptToConnectToRebootedDevice() returns a Device instantation
log(hostname,
"Beginning second post-reboot reconnect attempts")
secondReconnDev = attemptToConnectToRebootedDevice(
hostname, username, password)
# this means we timed out... device needs manual intervention
if secondReconnDev == False:
log(
hostname,
"SECOND Post-reboot FAILED. Device failed to come back up - Manual intervention required"
)
return {
'success':
0,
'message':
"SECOND Post-reboot FAILED. Device failed to come back up - Manual intervention required"
}
log(hostname, "Second post-reboot reconnection successful!")
log(
hostname,
"Checking if we booted from backup partition - show system storage partitions"
)
# Check if backup partition was booted
partition_check = secondReconnDev.rpc.get_system_storage_partitions(
)
partition_check = partition_check.xpath(
'//partitions/booted-from')
if len(partition_check) > 1:
return {
'success':
0,
'message':
"Couldn't determine active/backup partition from 'show system storage partitions'"
}
partition_check = str(partition_check[0].text).strip()
if partition_check == "backup":
log(
hostname,
"PROCEDURE FAILED, DEVICE STILL BOOTING FROM BACKUP PARTITION, DEVICE NEEDS TO BE RMA'D"
)
return {
'success':
0,
'message':
"PROCEDURE FAILED, DEVICE STILL BOOTING FROM BACKUP PARTITION, DEVICE NEEDS TO BE RMA'D"
}
# 3. Run the NAND media check after boot up
with StartShell(reconnDev) as reconnShell:
reconnShell.run('su root', this='Password:'******'nand-mediack -C')
log(hostname, 'Running nand-mediack -C')
if "nand-mediack -C\r\r\nMedia check on da0 on ex platforms\r\nroot@" not in nand_mediack[
1]:
log(hostname, 'nand-mediack check FAILED: ' + nand_mediack[1])
return {'success': 0, 'message': 'nand-mediack check FAILED'}
else:
log(hostname, 'nand-mediack check PASSED: ' + nand_mediack[1])
reconnDev.close()
log(hostname, "Completed execution")
log(hostname, "------------------------")
return {'success': 1, 'message': 'SCRIPT COMPLETED EXECUTION'}
def get_version(self):
ss = StartShell(self.dev)
ss.open()
self.version = ss.run('cli -c "show version"')
import filecmp
from shutil import copyfile
f = open("bak.router.list")
copyfile("result.log", "result.log.old")
res = open("result.log", "w")
for router in f:
router = router.replace("\n", "")
print("Checking Router: ", router)
dev = Device(host=router,
user='******',
password='******',
mode='telnet')
dev.open()
shell = StartShell(dev)
shell.open()
op = shell.run('cli -c "show system core-dumps | match root "')
for line in op:
if isinstance(line, str) and "root wheel" in line:
line = line.replace("%", "")
line = line[45:]
res.write(router + ":" + line)
dev.close()
res.close()
if os.stat("result.log").st_size and not filecmp.cmp("result.log",
"result.log.old"):
print("New Cores Generated")
content = "cat result.log | mail -s 'Google Escalation - New Cores Found' [email protected] [email protected] [email protected] [email protected]"
from jnpr.junos.utils.start_shell import StartShell
from jnpr.junos import Device
dev=Device('192.168.56.151', user='******', passwd='Juniper')
ss = StartShell(dev)
ss.open()
ss.run('pwd')
ss.run('cli -c "show version"')
ss.run('cli -c "show configuration | save localfile.txt"')
ss.close()
def _ssh_exec(self, command):
with StartShell(self._dev) as sh:
got = sh.run(command)
ok = sh.last_ok
return (ok, got)
channel = ssh.invoke_shell()
stdin, stdout, stderr = ssh.exec_command(
'file archive compress source /var/log/* destination /var/tmp/logfiles-' + date_arg + '.tgz\n')
exit_status = stdout.channel.recv_exit_status()
if exit_status == 0:
logging.info('Info: Logfiles compressed successfully.')
ssh.close()
else:
logging.info('Error: Logfiles not compressed successfully. Check Device manually.')
ssh.close()
else:
for i in vcmember_list:
dev_ntest = Device(host=varIP, user=varUser, password=varPassword,normalize=True)
try:
dev_ntest.open()
with StartShell(dev_ntest) as bsd:
bsd.run('rlogin -Ji fpc' + str(i), timeout=1)
bsd.wait_for(this='Password:'******'start shell', timeout=1)
bsd.wait_for(this='%', timeout=1)
bsd.run('tar -zcvf /var/tmp/varlog-mem' + str(i) + '.tar.gz /var/log/* ')
bsd.wait_for(this='%', timeout=1)
bsd.run('cli -c "file copy /var/tmp/varlog-mem' + str(i) + '.tar.gz fpc'+ str(varVCID_master) +':/var/tmp/"')
bsd.wait_for(this='%', timeout=1)
dev_ntest.close()
except:
logging.info('Error: VC-Logfiles not compressed successfully. Check Device manually.')
dev_ntest.close()
# Make single archive for VC
dev_ntest = Device(host=varIP, user=varUser, password=varPassword,normalize=True)