def _prepare_issu(self, steps, upgrade_image):
"""Prepare the device for ISSU:
1. Check currect image version and upgrade image version
2. Copy upgrade image to standby RP
Raises:
Unicon errors
Exception
Example:
>>> _prepare_issu(steps=steps, upgrade_image='someimage')
"""
# Init
device = self.device
filetransfer = FileUtils.from_device(device)
if not hasattr(self.device, 'filetransfer_attributes'):
filetransfer = FileUtils.from_device(self.device)
set_filetransfer_attributes(self, self.device, filetransfer)
for disk in ['harddisk:', 'stby-harddisk:']:
# Check for space on RP and SRP
logger.info("Verify '{}' has enough space".format(disk))
try:
self.check_disk_space(device=device,
disk=disk,
image=upgrade_image)
except Exception as e:
raise Exception(
"FAIL: Device '{}' does not have enough space -"
" skipping ISSU".format(disk))
# Copy ISSU upgrade image to disk
logger.info("Copy ISSU upgrade image to {}".format(disk))
from_url = '{protocol}://{address}/{upgrade_image}'.format(
protocol=device.filetransfer_attributes['protocol'],
address=device.filetransfer_attributes['server_address'],
upgrade_image=upgrade_image)
filetransfer.copyfile(source=from_url,
destination=disk,
device=device,
timeout_seconds='600')
# Verify location:<filename> exists
output = device.execute('dir {disk}{image}'.format(
disk=disk, image=basename(upgrade_image)))
if 'Error' not in output:
logger.info("Copied ISSU image to '{}'".format(disk))
else:
raise Exception(
"Unable to copy ISSU image to '{}'".format(disk))
def verify_enough_server_disk_space(device, protocol, server=None, directory='.',
required_space=None, timeout=300, fu_session=None):
"""Verify there are enough space on the server
Args:
device ('obj'): Device object
protocol ('str'): Protocol used to check disk space, scp or sftp
server ('str'): Server address or hostname. if not provided it will perform
operation on local file system (Optional)
required_space ('int'): required total disk space (Optional)
directory ('str'): directory to check file size (Optional)
timeout('int'): timeout in seconds (Optional, default 300)
fu_session ('obj'): existing FileUtils object to reuse (Optional)
Returns:
True if enough space, False otherwise
"""
if fu_session:
return _verify_enough_server_disk_space(device, protocol, server=server,
directory=directory,
required_space=required_space,
timeout=timeout,
fu_session=fu_session)
else:
with FileUtils(testbed=device.testbed) as fu:
return _verify_enough_server_disk_space(device, protocol, server=server,
directory=directory,
required_space=required_space,
timeout=timeout,
fu_session=fu)
def verify_file_size_stable_on_server(device, protocol, file, server=None, max_tries=3,
delay=2, timeout=300, fu_session=None):
"""Verify size stability of given file on the server
Args:
device ('obj'): Device object
server ('str'): Server address or hostname. if not provided it will perform
operation on local file system (Optional)
protocol ('str'): Protocol used to check file, ftp or sftp
file ('int'): file path
timeout ('int'): timeout in seconds
fu_session ('obj'): existing FileUtils object to reuse
max_tries ('int'): number of tries to check file stability, defaults 3
delay ('int'): time delay between tries in seconds, defaults 2
Returns:
True if file size is stable, false otherwise
"""
# global session
if fu_session:
return _verify_file_size_stable_on_server(device, protocol, file, server=server,
max_tries=max_tries, delay=delay,
timeout=timeout, fu_session=fu_session)
# no global session, establish a local one
else:
with FileUtils(testbed=device.testbed) as fu:
return _verify_file_size_stable_on_server(device, protocol, file, server=server,
max_tries=max_tries, delay=delay,
timeout=timeout, fu_session=fu)
def verify_file_exists_on_server(device, protocol, file, server=None, size=None,
timeout=300, fu_session=None, max_tries=1):
"""Verify there are enough space on the server
Args:
device ('obj'): Device object
protocol ('str'): Protocol used to check file, ftp or sftp
file ('int'): file path
server ('str'): Server address or hostname. if not provided it will perform
operation on local file system (Optional)
size ('int'): expected file size in bytes, if not provided will only check
file existence with name (Optional)
timeout('int'): timeout in seconds (Optional)
fu_session ('obj'): existing FileUtils object to reuse (Optional)
max_tries ('int;): max number of attempts (Optional)
Returns:
True if enough space, false otherwise
"""
# global session
if fu_session:
return _verify_file_exists_on_server(device, protocol, file, server=server, size=size,
timeout=timeout, fu_session=fu_session, max_tries=max_tries)
# no global session, establish a local one
else:
with FileUtils(testbed=device.testbed) as fu:
return _verify_file_exists_on_server(device, protocol, file, server=server, size=size,
timeout=timeout, fu_session=fu, max_tries=max_tries)
def stop(self):
# Get buffer
output = self.device.expect(".*")
# send cntrl+c
self.device.send('\x03')
# Bring back the pcap file if any
if self.pcap_file and self.local_dir:
# Copy it back to local host
# Find server
servers = self.device.testbed.servers
# Check if there is a self.protocol server
if self.protocol not in servers:
raise Exception("'{p}' server missing in the testbed "
"yaml file".format(p=self.protocol))
# Find ip
ip = servers[self.protocol]['address']
port = servers[self.protocol].get('custom', {}).get('port', 22)
local_file = os.path.join(self.local_dir,
os.path.basename(self.pcap_file))
# Create directory if doesnt exists
os.makedirs(self.local_dir, exist_ok=True)
with FileUtils(testbed=self.device.testbed) as futils:
futils.get_child(self.protocol)
futils.children[self.protocol].SSH_DEFAULT_PORT = port
futils.copyfile(source='{p}://{i}/{path}'.format(
p=self.protocol, i=ip, path=self.pcap_file),
destination=self.local_dir)
return output
def cleanup(self, ip_vm, filename, username_vm):
try:
with FileUtils(testbed=testbed) as futils:
futils.deletefile(
f'sftp://{ip_vm}/home/{username_vm}/{filename}')
futils.deletefile(
f'sftp://{ip_vm}/home/{username_vm}/received_from_host_{filename}'
)
# futils.deletefile(target=f'{self.host_path}/received_from_remotevm_{filename}')
# futils.deletefile(target=f'{self.host_path}/{filename}')
os.remove(f'{self.host_path}/received_from_remotevm_{filename}')
os.remove(f'{self.host_path}/{filename}')
print(
"=========================test files were deleted on both machines"
)
except:
print("Somthing went wrong while deleting a files...")
self.passx("I think it is passed")
try:
self.vm.disconnect_all()
except:
print("Somthing went wrong while disconnect all..")
self.passx("I think it is passed")
self.passed('Cleanup is done')
def delete_file_on_server(testbed,
server,
path,
protocol='sftp',
timeout=300,
fu_session=None):
""" delete the file from server
Args:
testbed ('obj'): testbed object containing the server info
server ('str"): server address or hostname
path ('str'): file path on server
protocol ('str'): protocol used for deletion, defaults to sftp
timeout ('int'): connection timeout
Returns:
None
"""
if fu_session:
return _delete_file_on_server(server,
path,
protocol=protocol,
timeout=timeout,
fu_session=fu_session)
else:
with FileUtils(testbed=testbed) as fu:
return _delete_file_on_server(server,
path,
protocol=protocol,
timeout=timeout,
fu_session=fu)
def upload_core_to_linux(self, core):
"""Upload core files to composed path.
Args:
Mandatory:
core (`str`) : Core file
Example:
>>> upload_core_to_linux(core='RP_0_bt_logger_13899_20180112-184444-EST.core.gz')
"""
# if the setup was not done because the configure subsection did
# not run, then we do the setup here
if not hasattr(self.device, 'filetransfer_attributes'):
filetransfer = FileUtils.from_device(self.device)
set_filetransfer_attributes(self, self.device, filetransfer)
filename, core = self.get_upload_cmd(**core)
message = "Core dump upload attempt: {}".format(filename)
from_URL = 'core:{core}'.format(core=core)
to_URL = '{protocol}://{address}/{path}/{filename}'.format(
protocol=self.device.filetransfer_attributes['protocol'],
address=self.device.filetransfer_attributes['server_address'],
path=self.device.filetransfer_attributes['path'],
filename=filename)
self.filetransfer.copyfile(device=self.device,
source=from_URL,
destination=to_URL)
def copy_from(self, env):
with FileUtils(testbed=env) as futils:
futils.copyfile(
source='scp://rrr//home/adminaccount/slavik/3.txt',
destination=
'/home/jsakhno/github/pyatsTraining/homeworks/yaroslav_sakhno/hw03'
)
def get_file_size_from_server(device,
server,
path,
protocol,
timeout=300,
fu_session=None):
""" Get file size from the server
Args:
device ('Obj'): Device object
server ('str'): server address or hostname
path ('str'): file path on server to check
protocol ('srt'): protocol used to check file size
timeout ('int'): check size timeout in seconds
fu_session ('obj'): existing FileUtils object to reuse
Returns:
integer representation of file size in bytes
"""
if fu_session:
return _get_file_size_from_server(server,
path,
protocol,
timeout=timeout,
fu_session=fu_session)
else:
with FileUtils(testbed=device.testbed) as fu:
return _get_file_size_from_server(server,
path,
protocol,
timeout=timeout,
fu_session=fu)
def save_configuration(self,
device,
method,
abstract,
default_dir,
copy_to_standby=False):
''' Save current configuration to a checkpoint file '''
if method == 'checkpoint' or method == 'config_replace':
# Create unique filename
self.to_url = self.__class__.__name__ + time.ctime().\
replace(' ', '_').\
replace(':', '_')
# Save configuration
self.create_checkpoint_file(device=device, file=self.to_url)
# Instantiate a filetransferutils instance for JunOS device
self.filetransfer = FileUtils.from_device(device)
# Verify checkpoint file exists
if self.to_url in self.filetransfer.dir(target=self.to_url,
device=device):
log.info("Successfully created checkpoint/file '{}'".\
format(self.to_url))
else:
raise Exception("Unable to create checkpoint/file '{}'".\
format(self.to_url))
# Return filename generated to caller
return self.to_url
else:
raise NotImplementedError("save configuration using method '{}'".\
format(method))
def save_configuration(self,
device,
method,
abstract,
default_dir,
copy_to_standby=False):
if method == 'checkpoint':
# compose checkpoint name
self.ckname = self.__class__.__name__ + \
time.ctime().replace(' ', '_').replace(':', '_')
# Create checkpoint
self.create_delete_checkpoint(device=device,
name=self.ckname,
abstract=abstract,
action='create')
# Check if checkpoint is successfully created
self.check_checkpoint_status(device=device,
name=self.ckname,
abstract=abstract)
elif method == 'local':
self.run_config = device.execute('show running-config')
elif method == 'config_replace':
# Create unique filename
self.filename = self.__class__.__name__ + \
time.ctime().replace(' ', '_').replace(':', '_')
# Set from/to locations
self.from_url = 'running-config'
self.to_url = '{dir}{filename}'.format(
filename=self.filename, dir=default_dir[device.name])
# Instantiate a filetransferutils instance for IOSXE device
self.filetransfer = FileUtils.from_device(device)
# Execute copy running-config to location:<filename>
self.filetransfer.copyconfiguration(source=self.from_url,
destination=self.to_url,
device=device)
if copy_to_standby:
self.stby_url = '{dir}{filename}'.format(
dir='stby-{}'.format(default_dir[device.name]),
filename=self.filename)
# copy config to stby-bootflash:
self.filetransfer.copyconfiguration(source=self.from_url,
destination=self.stby_url,
device=device)
# Verify location:<filename> exists
created = self.filetransfer.stat(target=self.to_url, device=device)
if created:
log.info("Successfully created '{}'".format(self.to_url))
else:
raise Exception("Unable to create '{}'".format(self.to_url))
# Return filename generated to caller
return self.to_url
def convert_server_to_linux_device(device, server):
"""
Args
converts a server block to a device object
device ('obj'): Device object
server ('str'): server hostname
Returns:
A Device object that can be connected
"""
with FileUtils(testbed=device.testbed) as fu:
server_block = fu.get_server_block(server)
hostname = fu.get_hostname(server)
device_obj = Device(
name=server,
os='linux',
credentials=server_block.credentials,
connections={'linux': {
'ip': hostname,
'protocol': 'ssh'
}},
custom={'abstraction': {
'order': ['os']
}},
type='linux',
testbed=device.testbed)
return device_obj
def _prepare_issu(self, steps, upgrade_image):
"""Prepare the device for ISSU:
1. Check currect image version and upgrade image version
2. Copy upgrade image to standby RP
NXOS:
1. Copy image onto the device
Raises:
Unicon errors
Exception
Example:
>>> _prepare_issu(steps=steps, upgrade_image='someimage')
"""
# # Init
device = self.device
if not hasattr(self.device, 'filetransfer_attributes'):
filetransfer = FileUtils.from_device(self.device)
set_filetransfer_attributes(self, self.device, filetransfer)
disk = "bootflash:"
timeout_seconds = 600
with steps.start('Check available diskspace') as step:
dir_output = filetransfer.parsed_dir(disk, timeout_seconds, Dir)
if int(dir_output['disk_free_space']) < 4500000000:
step.failed(
"Not enough free space available to copy over the image.Free up atleast 4.5GB of space on {}"
.format(disk))
with steps.start('Copy over the issu image') as step:
# Copy ISSU upgrade image to disk
from_url = '{protocol}://{address}/{upgrade_image}'.format(
protocol=device.filetransfer_attributes['protocol'],
address=device.filetransfer_attributes['server_address'],
upgrade_image=upgrade_image)
filetransfer.copyfile(source=from_url,
destination=disk,
device=device,
vrf='management',
timeout_seconds=600)
# Verify location:<filename> exists
output = device.execute('dir {disk}{image}'.format(
disk=disk, image=basename(upgrade_image)))
if 'No such file or directory' not in output:
log.info("Copied ISSU image to '{}'".format(disk))
else:
step.failed(
'Required ISSU image {} not found on disk. Transfer failed.'
.format(basename(upgrade_image)))
def copy_file_from_vm_to_host(self, ip_vm, filename, username_vm):
try:
with FileUtils(testbed=testbed) as futils:
futils.copyfile(
source=f'sftp://{ip_vm}/home/{username_vm}/{filename}',
destination=
f'{self.host_path}/received_from_remotevm_{filename}')
except:
print('while copying file from vm to host...something went wrong')
def test_fu_from_device_protocol(self):
# Instantiate a filetransferutils instance for each os and protocol
for os_ in ['ios', 'iosxe', 'iosxr', 'nxos']:
device = AttrDict(os=os_)
for proto in ['ftp', 'tftp', 'scp', 'sftp', 'http']:
fu = FileUtils.from_device(device, protocol=proto)
self.assertIn(
'genie.libs.filetransferutils.plugins.{os_}.{proto}.fileutils.FileUtils'
.format(os_=os_, proto=proto),
str(fu.__class__))
self.assertEqual(fu.protocol, proto)
self.assertEqual(fu.os, os_)
def save_configuration_to_file(self, device, default_dir, file_name):
''' Save current configuration to file on device
'''
file_path = '{}{}'.format(default_dir, file_name)
try:
# Instantiate a filetransferutils instance for IOSXE device
self.filetransfer = FileUtils.from_device(device)
self.filetransfer.copyconfiguration(source='running-config',
destination=file_path,
device=device)
except Exception as e:
log.error(e)
raise Exception(
"Issue saving config to {c}".format(c=file_path)) from e
def delete_unprotected_files(device,
directory,
protected,
files_to_delete=None,
dir_output=None):
"""delete all files not matching regex in the protected list
Args:
device ('obj'): Device object
directory ('str'): working directory to perform the operation
protected ('list'): list of file patterns that won't be deleted. If it begins
and ends with (), it will be considered as a regex
files_to_delete('list') list of files that should be deleted unless they are not protected
dir_output ('str'): output of dir command, if not provided execute the cmd on device to get the output
Returns:
None
"""
fu_device = FileUtils.from_device(device)
file_set = set(
Dq(device.parse('ls -l {}'.format(directory),
output=dir_output)).get_values('files'))
protected_set, not_protected = _protected_and_unprotected_files(
file_set, protected, files_to_delete)
error_messages = []
if not_protected:
log.info("The following files will be deleted:\n{}".format(
'\n'.join(not_protected)))
dont_delete_list = protected_set.intersection(files_to_delete)
if dont_delete_list:
log.info(
"The following files will not be deleted because they are protected:\n{}"
.format('\n'.join(dont_delete_list)))
for file in not_protected:
# it's a directory, dont delete
if file.endswith('/'):
continue
log.info('Deleting the unprotected file "{}"'.format(file))
try:
fu_device.deletefile(file, device=device)
except Exception as e:
error_messages.append('Failed to delete file "{}" due '
'to :{}'.format(file, str(e)))
if error_messages:
raise Exception('\n'.join(error_messages))
else:
log.info(
"No files will be deleted, the following files are protected:\n{}".
format('\n'.join(protected_set)))
def copy_to_device(device,
remote_path,
local_path,
server,
protocol,
vrf=None,
timeout=300,
compact=False,
use_kstack=False,
username=None,
password=None,
**kwargs):
"""
Copy file from linux server to device
Args:
device ('Device'): Device object
remote_path ('str'): remote file path on the server
local_path ('str'): local file path to copy to on the device
server ('str'): hostname or address of the server
protocol('str'): file transfer protocol to be used
vrf ('str'): vrf to use (optional)
timeout('int'): timeout value in seconds, default 300
compact('bool'): compress image option for n9k, defaults False
username('str'): Username to be used during copy operation
password('str'): Password to be used during copy operation
use_kstack('bool'): Use faster version of copy, defaults False
Not supported with a file transfer protocol
prompting for a username and password
Returns:
None
"""
# aci uses the linux implementation and fileutils doesnt support
# os/platform abstraction
setattr(device, 'os', 'linux')
fu = FileUtils.from_device(device)
setattr(device, 'os', 'nxos')
generic_copy_to_device(device=device,
remote_path=remote_path,
local_path=local_path,
server=server,
protocol=protocol,
vrf=vrf,
timeout=timeout,
compact=compact,
use_kstack=use_kstack,
username=username,
password=password,
fu=fu,
**kwargs)
def copy_issu_image_to_disk(device,
disk,
path,
address,
image,
protocol="tftp",
timeout_seconds=600,
wait_time_after_copy=0,
overwrite=False):
""" Copy image from a server to disk
Args:
device ('obj'): Device object
disk ('str'): Disk name
address ('str'): Server address
path ('str'): Path on server
protocol ('str'): Transfer protocol
image ('str'): Image name
timeout_seconds ('int'): Maximum duration to wait for file copy
wait_time_after_copy ('int'): Wait time after file copy
overwrite ('bool'): Flag to overwrite existing file
Raises:
Exception: Failed copying ISSU image to disk
Returns:
None
"""
from_url = "{protocol}://{address}/{path}/{image}".format(
protocol=protocol, address=address, path=path, image=image)
filetransfer = FileUtils.from_device(device)
filetransfer.copyfile(source=from_url,
destination=disk,
device=device,
timeout_seconds=timeout_seconds,
overwrite=overwrite)
time.sleep(wait_time_after_copy)
output = device.execute("dir {disk}{image}".format(disk=disk,
image=basename(image)))
if "Error" not in output:
log.info("Copied ISSU image to '{disk}'".format(disk=disk))
else:
raise Exception(
"Unable to copy ISSU image to '{disk}'".format(disk=disk))
def copy_to_server(testbed,
protocol,
server,
local_path,
remote_path,
timeout=300,
fu_session=None,
quiet=False):
""" Copy file from directory to server
Args:
testbed ('obj'): Testbed object
protocol ('str'): Transfer protocol
server ('str'): Server name in testbed yaml or server ip address
local_path ('str'): File to copy, including path
remote_path ('str'): Where to save the file, including file name
timeout('int'): timeout value in seconds, default 300
fu_session ('obj'): existing FileUtils object to reuse
quiet ('bool'): quiet mode -- does not print copy progress
Returns:
None
Raises:
Exception
"""
if fu_session:
_copy_to_server(protocol,
server,
local_path,
remote_path,
timeout=timeout,
fu_session=fu_session,
quiet=quiet)
else:
with FileUtils(testbed=testbed) as fu:
_copy_to_server(protocol,
server,
local_path,
remote_path,
timeout=timeout,
fu_session=fu,
quiet=quiet)
def copy_to_server(testbed, protocol, server, local_path, remote_path):
""" Copy file from directory to server
Args:
testbed ('obj'): Testbed object
protocol ('str'): Transfer protocol
server ('str'): Server name in testbed yaml or server ip address
local_path ('str'): File to copy, including path
remote_path ('str'): Where to save the file, including file name
Returns:
None
Raises:
Exception
"""
try:
# Check if server argument is an IP address
IPAddress(server)
except Exception:
try:
# Check if server argument is a server defined in testbed yaml
server = testbed.servers[server]["address"]
except Exception:
raise Exception(
"> For 'server' argument either provide an ip address\n"
"> OR\n"
"> configure the following in your testbed yaml and provide <protocol>:\n"
"> --------------------------------\n"
"> testbed:\n"
"> servers:\n"
"> <protocol>:\n"
"> address: <address>\n"
"> --------------------------------")
# Building remote address
remote = "{p}://{s}:/{f}".format(p=protocol, s=server, f=remote_path)
log.info("Copying {local_path} to {remote_path}".format(
local_path=local_path, remote_path=remote))
with FileUtils(testbed=testbed) as futils:
futils.copyfile(source=local_path, destination=remote)
def check_file_existense(self, filename, ip_vm, username_vm):
try:
if os.path.exists(
f'{self.host_path}/received_from_remotevm_{filename}'
) and self.vm.execute(
f'ls -la /home/pyast/received_from_host_{filename}'):
print(
'*****************************************************************File is received on host comp'
)
with FileUtils(testbed=testbed) as futils:
if futils.checkfile(
f'sftp://{ip_vm}/home/{username_vm}/received_from_host_{filename}'
):
print(
'**************************************************************File is received on remote vm'
)
else:
raise Exception
except:
print("Something went wrong while checking copied files")
self.passed('Files are copied correctly')
def copy_from_device(device,
remote_path,
local_path,
server,
protocol,
vrf=None,
timeout=300,
**kwargs):
"""
Copy file from device to linux server (Works for sftp and ftp)
Args:
device ('Device'): Device object
remote_path ('str'): remote file path to copy to on the server
local_path ('str'): local file path to copy from the device
server ('str'): hostname or address of the server
protocol('str'): file transfer protocol to be used
vrf ('str'): vrf to use (optional)
timeout('int'): timeout value in seconds, default 300
Returns:
None
"""
fu = FileUtils.from_device(device)
# build the source address
destination = '{p}://{s}/{f}'.format(p=protocol, s=server, f=remote_path)
if vrf:
fu.copyfile(source=local_path,
destination=destination,
device=device,
vrf=vrf,
timeout_seconds=timeout,
**kwargs)
else:
fu.copyfile(source=local_path,
destination=destination,
device=device,
timeout_seconds=timeout,
**kwargs)
class test_filetransferutils(unittest.TestCase):
# Instantiate tesbed and device objects
tb = Testbed(name='myTestbed')
device = Device(testbed=tb, name='aDevice', os='junos')
# Instantiate a filetransferutils instance for Junos device
fu_device = FileUtils.from_device(device)
# Add testbed servers for authentication
device.testbed.servers = AttrDict(server_name=dict(username="******",
password="******",
address='1.1.1.1'), )
# Mock device output
raw1 = '''
file copy golden_config ftp://[email protected]:/test/
Password for [email protected]:
ftp://[email protected]:/test/golden_config 100% of 3040 B 11 MBps
'''
outputs = {}
outputs['file copy golden_config ftp://[email protected]:/test/']\
= raw1
def mapper(self, key, timeout=None, reply=None, prompt_recovery=False):
return self.outputs[key]
def test_copyfile(self):
self.device.execute = Mock()
self.device.execute.side_effect = self.mapper
# Call copyfiles
self.fu_device.copyfile(source='golden_config',
destination='ftp://1.1.1.1:/test/',
timeout_seconds='300',
device=self.device)
class test_filetransferutils(unittest.TestCase):
# Instantiate tesbed and device objects
tb = Testbed(name='myTestbed')
device = Device(testbed=tb, name='aDevice', os='iosxe')
# Instantiate a filetransferutils instance for IOSXE device
fu_device = FileUtils.from_device(device)
# Add testbed servers for authentication
device.testbed.servers = AttrDict(server_name=dict(username="******",
password="******",
address='1.1.1.1'), )
dir_output = [
'flash:/nvram_config', 'flash:/.rollback_timer', 'flash:/memleak.tcl',
'flash:/bootloader_evt_handle.log', 'flash:/ISSUCleanGolden',
'flash:/gs_script', 'flash:/.prst_sync', 'flash:/nvram_config_bkup',
'flash:/tech_support', 'flash:/dc_profile_dir',
'flash:/RestoreTue_Mar_20_12_19_11_2018-Mar-20-11-20-09.900-0',
'flash:/vlan.dat', 'flash:/core', 'flash:/tools', 'flash:/CRDU',
'flash:/.dbpersist',
'flash:/RestoreTue_Mar_20_12_13_39_2018-Mar-20-11-14-38.106-0',
'flash:/iox', 'flash:/onep', 'flash:/boothelper.log',
'flash:/stby-vlan.dat', 'flash:/.installer'
]
# Mock device output
raw1 = '''
copy flash:/memleak.tcl ftp://1.1.1.1//auto/tftp-ssr/memleak.tcl
Address or name of remote host [1.1.1.1]?
Destination filename [/auto/tftp-ssr/memleak.tcl]?
!!
104260 bytes copied in 0.396 secs (263283 bytes/sec)
'''
raw2 = '''
Directory of flash:/
69698 drwx 4096 Mar 20 2018 10:25:11 +00:00 .installer
69720 -rw- 2097152 Mar 20 2018 13:09:24 +00:00 nvram_config
69700 -rw- 90761 Mar 20 2018 10:25:27 +00:00 bootloader_evt_handle.log
69701 drwx 4096 Feb 1 2018 13:44:32 +00:00 core
15489 drwx 4096 Mar 20 2018 10:31:08 +00:00 .prst_sync
30977 drwx 4096 May 2 2016 07:58:53 +00:00 .rollback_timer
38722 drwx 4096 Mar 20 2018 10:25:43 +00:00 dc_profile_dir
69699 -rw- 76 Mar 20 2018 10:25:46 +00:00 boothelper.log
69705 -rw- 104260 Mar 20 2018 10:26:01 +00:00 memleak.tcl
69706 drwx 4096 May 2 2016 08:11:23 +00:00 onep
69714 drwx 4096 Aug 13 2016 08:55:12 +00:00 iox
69734 -rw- 3496 Mar 11 2018 17:40:26 +00:00 vlan.dat
69708 -rw- 617329255 Sep 27 2017 09:11:39 +00:00 ISSUCleanGolden
69709 drwx 4096 Aug 3 2016 08:07:47 +00:00 gs_script
69712 drwx 4096 Mar 19 2017 09:26:23 +00:00 tools
69719 drwx 4096 Feb 12 2018 11:20:01 +00:00 .dbpersist
69703 -rw- 2097152 Mar 20 2018 13:09:25 +00:00 nvram_config_bkup
69729 -rw- 3496 Feb 12 2018 12:51:01 +00:00 stby-vlan.dat
69735 -rw- 27145 Mar 20 2018 11:14:45 +00:00 RestoreTue_Mar_20_12_13_39_2018-Mar-20-11-14-38.106-0
69721 drwx 4096 Sep 25 2017 07:59:54 +00:00 CRDU
69727 drwx 4096 Oct 23 2017 13:40:11 +00:00 tech_support
69736 -rw- 27145 Mar 20 2018 11:20:16 +00:00 RestoreTue_Mar_20_12_19_11_2018-Mar-20-11-20-09.900-0
1621966848 bytes total (906104832 bytes free)
'''
raw3 = '''
delete flash:memleak.tcl
Delete filename [memleak.tcl]?
Delete flash:/memleak.tcl? [confirm]
'''
raw4 = '''
rename flash:memleak.tcl new_file.tcl
Destination filename [new_file.tcl]?
'''
raw5 = '''
show clock | redirect ftp://1.1.1.1//auto/tftp-ssr/show_clock
Writing /auto/tftp-ssr/show_clock
'''
raw6 = {
'futlinux.check_file.return_value': '',
'futlinux.deletefile.return_value': ''
}
raw7 = '''
copy running-config tftp://10.1.7.250//auto/tftp-ssr/test_config.py
Address or name of remote host [10.1.7.250]?
Destination filename [/auto/tftp-ssr/test_config.py]?
!!
27092 bytes copied in 6.764 secs (4005 bytes/sec)
'''
outputs = {}
outputs['copy flash:/memleak.tcl ftp://1.1.1.1//auto/tftp-ssr/memleak.tcl']\
= raw1
outputs['dir'] = raw2
outputs['delete flash:memleak.tcl'] = raw3
outputs['rename flash:memleak.tcl new_file.tcl'] = raw4
outputs['show clock | redirect ftp://1.1.1.1//auto/tftp-ssr/show_clock'] = \
raw5
outputs['copy running-config tftp://10.1.7.250//auto/tftp-ssr/test_config.py'] = \
raw7
def mapper(self, key, timeout=None, reply=None, prompt_recovery=False):
return self.outputs[key]
def test_copyfile(self):
self.device.execute = Mock()
self.device.execute.side_effect = self.mapper
# Call copyfiles
self.fu_device.copyfile(
source='flash:/memleak.tcl',
destination='ftp://1.1.1.1//auto/tftp-ssr/memleak.tcl',
timeout_seconds='300',
device=self.device)
def test_dir(self):
self.device.execute = Mock()
self.device.execute.side_effect = self.mapper
directory_output = self.fu_device.dir(target='flash:',
timeout_seconds=300,
device=self.device)
self.assertEqual(sorted(directory_output), sorted(self.dir_output))
def test_stat(self):
self.device.execute = Mock()
self.device.execute.side_effect = self.mapper
file_details = self.fu_device.stat(target='flash:memleak.tcl',
timeout_seconds=300,
device=self.device)
self.assertEqual(file_details['last_modified_date'],
'Mar 20 2018 10:26:01 +00:00')
self.assertEqual(file_details['permissions'], '-rw-')
self.assertEqual(file_details['index'], '69705')
self.assertEqual(file_details['size'], '104260')
def test_deletefile(self):
self.device.execute = Mock()
self.device.execute.side_effect = self.mapper
self.fu_device.deletefile(target='flash:memleak.tcl',
timeout_seconds=300,
device=self.device)
def test_renamefile(self):
self.device.execute = Mock()
self.device.execute.side_effect = self.mapper
self.fu_device.renamefile(source='flash:memleak.tcl',
destination='new_file.tcl',
timeout_seconds=300,
device=self.device)
@patch(
'genie.libs.filetransferutils.plugins.fileutils.FileUtils.validateserver',
return_value=raw6)
def test_validateserver(self, raw6):
self.device.execute = Mock()
self.device.execute.side_effect = self.mapper
self.fu_device.validateserver(
target='ftp://1.1.1.1//auto/tftp-ssr/show_clock',
timeout_seconds=300,
device=self.device)
def test_copyconfiguration(self):
self.device.execute = Mock()
self.device.execute.side_effect = self.mapper
self.fu_device.copyconfiguration(
source='running-config',
destination='tftp://10.1.7.250//auto/tftp-ssr/test_config.py',
timeout_seconds=300,
device=self.device)
def _perform_issu(self, steps, upgrade_image, timeout=300):
"""Perform the ISSU steps in sequence on the ASR1K device:
1. Execute 'issu loadversion' to begin ISSU process
2. Poll until standby RP reaches 'ok' state
3. Verify ISSU state is now 'loadversion'
4. Execute 'issu runversion' to initiate RP failover
5. Reconnect to the device
6. Verify ISSU state is now 'runversion'
7. Execute 'issu acceptversion' to cancel rollback timer
8. Verify ISSU state is now 'acceptversion'
9. Verify ISSU rollback timer has been cancelled
10. Poll until standby RP reaches 'ok' state
11. Save running-configuration to startup-configuration
12. Execute 'issu commitversion' to complete ISSU process
13. Reload the device and then reconnect to it
14. Verify device is now booted with ISSU upgrade image
Raises:
Unicon errors
Exception
Example:
>>> _perform_issu(steps=steps, upgrade_image='someimage')
"""
# Init
device = self.device
lookup = Lookup.from_device(device)
filetransfer = FileUtils.from_device(device)
image_name = basename(upgrade_image)
# ======================================================================
# Get standby RP
# ======================================================================
with steps.start("Get standby RP information", continue_=True) as step:
platform_dict = lookup.parser.show_platform.\
ShowPlatform(device=device).parse()
# Standby RP
rs = R([
'slot', '(?P<val1>.*)', 'rp', '(?P<val2>.*)', 'state',
'ok, standby'
])
ret = find([platform_dict], rs, filter_=False, all_keys=True)
if not ret:
raise Exception(
"Device '{}' does not have standby RP - cannot "
"perform ISSU".format(device.name))
standby_rp = ret[0][1][1]
srp = re.search('(?P<srp>(\d))', standby_rp).groupdict()['srp']
logger.info("Standby RP on '{dev}' is: '{standby_rp}'".format(
dev=device.name, standby_rp=standby_rp))
# ======================================================================
# issu loadversion
# ======================================================================
with steps.start("Execute 'issu loadversion' to begin ISSU process",
continue_=True) as step:
try:
output = device.execute('issu loadversion rp {srp} file '
'stby-harddisk:{image}'.format(
srp=srp, image=image_name),
timeout=600)
if 'FAILED' in output:
device.execute('issu abortversion', timeout=timeout)
raise Exception("Unable to execute 'issu loadversion'")
except Exception as e:
raise Exception("Unable to execute 'issu loadversion'")
# Poll until standby RP reaches 'ok' state in 'show platform'
logger.info("Poll until standby RP reaches 'ok' state")
platform_timeout = Timeout(max_time=1200, interval=120)
while platform_timeout.iterate():
platform_dict = lookup.parser.show_platform.\
ShowPlatform(device=device).parse()
# Create requirement to find standby-RP with 'ok, standby' state
rs = R([
'slot', '(?P<val1>.*)', 'rp', '(?P<val2>.*)', 'state',
'ok, standby'
])
ret = find([platform_dict], rs, filter_=False, all_keys=True)
if ret:
logger.info("Stanby RP '{}' is in 'ok' state".\
format(standby_rp))
break
# Standby RP is not 'ok' state as yet, sleep and recheck
platform_timeout.sleep()
# Verify issu state
logger.info("Verify ISSU state is now 'loadversion'")
try:
self.check_issu_state(device=device,
slot=standby_rp,
expected_state='loadversion')
logger.info("ISSU state is 'loadversion' as exepcted")
except Exception as e:
raise Exception(str(e))
# ======================================================================
# issu runversion
# ======================================================================
with steps.start("Execute 'issu runversion' to initiate RP failover",
continue_=True) as step:
try:
output = device.execute('issu runversion', timeout=timeout)
except SubCommandFailure:
# Timeout Unicon SubCommandFailure expected
# Wait a bit as the device is booting with the ISSU upgrade image
time.sleep(timeout)
pass
# Reconnect to device
logger.info("Reconnect to the device after runversion")
reconnect_timeout = Timeout(max_time=1200, interval=120)
self._reconnect(steps=steps, timeout=reconnect_timeout)
# Verify issu state
logger.info("Verify ISSU state is now 'runversion'")
try:
self.check_issu_state(device=device,
slot=standby_rp,
expected_state='runversion')
logger.info("ISSU state is 'runversion' as exepcted")
except Exception as e:
raise Exception(str(e))
# ======================================================================
# issu acceptversion
# ======================================================================
with steps.start(
"Execute 'issu acceptversion' to cancel rollback timer",
continue_=True) as step:
try:
output = device.execute('issu acceptversion', timeout=timeout)
if 'FAILED' in output:
raise Exception("Unable to execute 'issu acceptversion'")
except Exception as e:
raise Exception("Unable to execute 'issu acceptversion'",
from_exception=e)
# Verify issu state
logger.info("Verify ISSU state is now 'acceptversion'")
try:
self.check_issu_state(device=device,
slot=standby_rp,
expected_state='acceptversion')
logger.info("ISSU state is 'acceptversion' as exepcted")
except Exception as e:
raise Exception(str(e))
# Verify rollback timer
logger.info("Verify ISSU rollback timer is now 'inactive'")
try:
self.check_issu_rollback_timer(device=device,
slot=standby_rp,
expected_state='inactive')
logger.info("ISSU rollback timer is 'inactive' as exepcted")
except Exception as e:
raise Exception(str(e))
# Poll until standby RP reaches 'ok' state in 'show platform'
logger.info("Poll until standby RP reaches 'ok' state")
platform_timeout = Timeout(max_time=1200, interval=120)
while platform_timeout.iterate():
platform_dict = lookup.parser.show_platform.\
ShowPlatform(device=device).parse()
# Create requirement to find standby-RP with 'ok, standby' state
rs = R([
'slot', '(?P<val1>.*)', 'rp', '(?P<val2>.*)', 'state',
'ok, standby'
])
ret = find([platform_dict], rs, filter_=False, all_keys=True)
if ret:
logger.info("Stanby RP '{}' is in 'ok' state".\
format(standby_rp))
break
# Standby RP is not 'ok' state as yet, sleep and recheck
platform_timeout.sleep()
# Save running-configuration to startup-configuration
logger.info("Save running-configuration to startup-configuration")
filetransfer.copyconfiguration(source='running-config',
destination='startup-config',
device=device)
# ======================================================================
# issu commitversion
# ======================================================================
with steps.start("Execute 'issu commitversion'",
continue_=True) as step:
try:
output = device.execute('issu commitversion', timeout=timeout)
if 'FAILED' in output:
raise Exception("Unable to execute 'issu commitversion'")
except Exception as e:
raise Exception("Unable to execute 'issu commitversion'",
from_exception=e)
# ======================================================================
# reload device
# ======================================================================
try:
reload_timeout = Timeout(max_time=1200, interval=120)
self.reload(steps=steps, timeout=reload_timeout)
except Exception as e:
raise Exception("Unable to reload the device after ISSU completed",
from_exception=e)
# ======================================================================
# verify image version
# ======================================================================
with steps.start(
"Verify device is loaded with upgraded image after ISSU",
continue_=True) as step:
try:
output = device.execute('show version | i image')
if image_name in output:
logger.info("ISSU upgrade image is successfully loaded on "
"the device '{}'".format(device.name))
except Exception as e:
raise Exception("Unable to execute 'show version'",
from_exception=e)
def perform_issu(device,
image,
disk,
timeout=1200,
reconnect_via=None,
steps=Steps()):
""" Execute ISSU on device
Args:
device ('obj'): Device object
image ('str'): Image name on disk
disk ('str'): Disk where is located image
timeout ('int'): Timeout in second for each section
Raise:
None
Returns:
None
"""
with steps.start("Command 'issu loadversion'") as step:
slot_number = get_platform_standby_rp(device=device)
if not slot_number:
raise ValueError("Could not retrieve standby rp slot number")
# Load version
standby_slot = "R{}".format(slot_number)
try:
issu_loadversion(device=device,
standby_slot=slot_number,
disk=disk,
image=image,
timeout=timeout)
except Exception:
step.failed("Unable to execute 'issu loadversion'")
with steps.start("Command 'issu runversion'") as step:
if not is_platform_slot_in_state(
device=device, slot=standby_slot, state="ok, standby"):
step.failed("Slot {slot} is not in 'ok, standby' state".format(
slot=standby_slot))
if not is_issu_terminal_state_reached_on_slot(device=device,
slot=standby_slot):
step.failed("Slot {slot} has not reached terminal state".format(
slot=standby_slot))
# Run version
try:
issu_runversion(device=device,
timeout=timeout,
reconnect_via=reconnect_via)
except (Exception, ConnectionError) as e:
step.failed(e)
with steps.start("Command 'issu acceptversion'") as step:
in_state = is_issu_in_state(device=device,
slot=standby_slot,
expected_state="runversion")
if not in_state:
step.failed("Issu is not in state 'runversion'")
# Accept version
try:
issu_acceptversion(device=device, timeout=timeout)
except Exception as e:
step.failed(e)
with steps.start(
"Save running-configuration to startup-configuration") as step:
filetransfer = FileUtils.from_device(device)
filetransfer.copyconfiguration(
source="running-config",
destination="startup-config",
device=device,
)
with steps.start("Command 'issu commitversion'") as step:
in_state = is_issu_in_state(device=device,
slot=standby_slot,
expected_state="acceptversion")
if not in_state:
step.failed("Issu is not in state 'acceptversion'")
in_state = is_issu_rollback_timer_in_state(device=device,
slot=standby_slot,
expected_state="inactive")
if not in_state:
step.failed("Issu rollback timer is not 'inactive'")
# Commit version
try:
issu_commitversion(device=device, timeout=timeout)
except Exception as e:
step.failed(e)
with steps.start("Reload standby slot") as step:
slot_number = get_platform_standby_rp(device=device)
if not slot_number:
raise ValueError("Could not retrieve standby rp slot number")
standby_slot = "R{}".format(slot_number)
try:
reload_issu_slot(device=device, slot=standby_slot, timeout=timeout)
except Exception as e:
step.failed(e)
def delete_unprotected_files(device,
directory,
protected,
files_to_delete=None,
dir_output=None,
allow_failure=False):
""" Delete all files not matching regex in the protected list
Args:
device ('obj'): Device object
directory ('str'): working directory to perform the operation
protected ('list'): list of file patterns that won't be deleted. If it begins
and ends with (), it will be considered as a regex
files_to_delete('list') list of files that should be deleted unless they are not protected
dir_output ('str'): output of dir command, if not provided execute the cmd on device to get the output
allow_failure (bool, optional): Allow the deletion of a file to silently fail. Defaults to False.
Returns:
None
"""
protected_set = set()
fu_device = FileUtils.from_device(device)
file_set = set(
device.parse('dir {}'.format(directory),
output=dir_output).get('files', {}).keys())
if isinstance(protected, str):
protected = [protected]
elif not isinstance(protected, (list, set)):
raise TypeError("'{p}' must be a list")
for pattern in protected:
# it's a regex!
if pattern.startswith('(') and pattern.endswith(')'):
regexp = re.compile(pattern)
protected_set.update(set(filter(regexp.match, file_set)))
# just file names, exact match only
elif pattern in file_set:
protected_set.add(pattern)
# if files_to_delete is given,updated protected files with the diff of file_set - files_to_delete
# so that we only delete files that are in files_to_delete and NOT protected
# in other words we remove the protected files from file_to_delete
if files_to_delete:
protected_set.update(file_set - set(files_to_delete))
not_protected = file_set - protected_set
error_messages = []
log.debug("protected patterns : {}".format(protected))
log.debug("found files : {}".format(file_set))
log.debug("protected files : {}".format(protected_set))
log.debug("non-protected files : {}".format(not_protected))
if not_protected:
log.info("The following files will be deleted:\n{}".format(
'\n'.join(not_protected)))
dont_delete_list = protected_set.intersection(files_to_delete)
if dont_delete_list:
log.info(
"The following files will not be deleted because they are protected:\n{}"
.format('\n'.join(dont_delete_list)))
for file in not_protected:
# it's a directory, dont delete
if file.endswith('/'):
continue
log.info('Deleting the unprotected file "{}"'.format(file))
try:
fu_device.deletefile(directory + file, device=device)
except Exception as e:
if allow_failure:
log.info(
'Failed to delete file "{}" but ignoring and moving '
'on due to "allow_failure=True"..'.format(file))
continue
error_messages.append('Failed to delete file "{}" due '
'to :{}'.format(file, str(e)))
if error_messages:
raise Exception('\n'.join(error_messages))
else:
log.info(
"No files will be deleted, the following files are protected:\n{}".
format('\n'.join(protected_set)))
def restore_configuration(self,
device,
method,
abstract,
iteration=10,
interval=60,
compare=False,
compare_exclude=[],
reload_timeout=1200):
if method == 'checkpoint':
# Enable the feature
for i in range(1, iteration):
try:
out = self.rollback_checkpoint(device=device,
name=self.ckname)
except Exception as e:
raise Exception('Unable to rollback config')
if out and 'Rollback Done' in out:
break
else:
log.info('Rollback configuration failed: sleeping {} '
'seconds and retrying...'.format(interval))
time.sleep(interval)
else:
raise Exception('Unable to rollback config')
# Delete the checkpoint
self.create_delete_checkpoint(device=device,
name=self.ckname,
abstract=abstract,
action='delete')
# Check if checkpoint is successfully deleted
self.check_checkpoint_status(device=device,
name=self.ckname,
expect='delete',
abstract=abstract)
elif method == 'local':
# reover the deivce with whole running-config
device.configure(self.run_config)
elif method == 'config_replace':
# delete the archive file
dialog = Dialog([
Statement(pattern=r'This will apply all necessary.*',
action='sendline(Y)',
loop_continue=True,
continue_timer=False),
Statement(pattern=r'less than running config.*',
action='sendline(Y)',
loop_continue=True,
continue_timer=False),
])
# dialog for plan B
alt_dialog = Dialog([
Statement(pattern=r'Destination filename.*',
action='sendline()',
loop_continue=True,
continue_timer=False),
])
for i in range(1, iteration):
# configure replace location:<filename>
out = device.execute('configure replace {}'.\
format(self.to_url), reply=dialog, error_pattern=[])
if out and 'Rollback Done' in out:
break
elif 'invalid' in out.lower():
# device does not support config replace, do it in different way
device.execute('erase startup-config')
device.execute('copy {} startup-config'.format(
self.to_url),
reply=alt_dialog)
device.reload(reload_timeout=reload_timeout)
break
else:
log.info(
'Config replace failed: sleeping {} seconds before'
' retrying.'.format(interval))
time.sleep(interval)
else:
raise Exception('Unable to execute config replace')
# Compare restored configuration to details in file
if compare:
log.info(
"Comparing current running-config with config-replace file"
)
# Default
exclude = [
'device', 'maker', 'diff_ignore', 'callables',
'(Current configuration.*)',
'(.*Building configuration.*)', '(.*Load for.*)',
'(.*Time source.*)'
]
if compare_exclude:
if isinstance(compare_exclude, str):
exclude.extend([compare_exclude])
else:
exclude.extend(compare_exclude)
# show run
show_run_output = device.execute('show running-config')
show_run_config = Config(show_run_output)
show_run_config.tree()
# location:<filename> contents
more_file = device.execute('more {}'.format(self.to_url))
more_file_config = Config(more_file)
more_file_config.tree()
# Diff 'show run' and config replace file contents
diff = Diff(show_run_config.config,
more_file_config.config,
exclude=exclude)
diff.findDiff()
# Check for differences
if len(diff.diffs):
log.error("Differences observed betweenrunning-config and "
"config-replce file:'{f}' for device {d}:".\
format(f=self.to_url, d=device.name))
log.error(str(diff.diffs))
raise Exception(
"Comparison between running-config and "
"config-replace file '{f}' failed for device"
" {d}".format(f=self.to_url, d=device.name))
else:
log.info("Comparison between running-config and config-replace"
"file '{f}' passed for device {d}".\
format(f=self.to_url, d=device.name))
# Delete location:<filename>
self.filetransfer = FileUtils.from_device(device)
self.filename = self.to_url
self.filetransfer.deletefile(target=self.to_url, device=device)
# Verify location:<filename> deleted
dir_output = self.filetransfer.dir(target=self.to_url,
device=device)
for file in dir_output:
if self.filename in file:
break
else:
log.info("Successfully deleted '{}'".format(self.to_url))
return
raise Exception("Unable to delete '{}'".format(self.to_url))
else:
# modify the device via callable function
# using Conf object
self.modify_func(device=device,
conf=self.conf,
values_dict=self.conf_argument,
recover=True,
**self.specific_args)
