def _extra_core(self, steps):
with steps.start('Verify if no extra core has been generated',
continue_=True) as step:
# Check if core of process is found
filetransfer = self.device.filetransfer if \
hasattr(self.device, 'filetransfer') else None
ha = self.abstract.sdk.libs.abstracted_libs.ha.HA(
device=self.device, filetransfer=filetransfer)
temp = TempResult(container=step)
cores = ha.check_cores()
for core in cores:
# Upload core of all the other core found
if int(core['pid']) == self.previous_pid:
continue
# Different core than the one expected!
log.error("Process '{p}' with '{pp}' has created a core "\
"but should not have".\
format(p=core['process'],
pp=core['pid']))
temp.failed('More core than expected has been created')
try:
ha.upload_core_to_linux(core)
except Exception as e:
log.error('Cannot upload core files\n{}'.format(str(e)))
# Clear cores
ha.clear_cores()
temp.result()
def _verify_restart_no_sysmgr(self, repeat_restart, steps, timeout):
'''Verify with show process that the pid has changed'''
# TODO
# Verify if timestamps is increasing
# TO BE TESTED WHEN SWITCH TO UNICON
# with steps.start('Verify the timestamp "previous < current"',
# continue_=True) as step:
# # get timestamp
# timestamp = self.get_process_timestamp(process=self.process)
# if timestamp <= self.previous_timestamp:
# temp.failed("The timestamp is not increasing after restart "
# "process '{p}'".format(p=self.process))
# else:
# temp.passed('The timestamp incresaed')
# Verify if the pid is not the same
with steps.start('Verify if the pid is the same',
continue_=True) as step:
temp = TempResult(container=step)
while timeout.iterate():
if self.helper:
previous_pid, _ = self._process_information_no_sysmgr(
self.helper)
else:
previous_pid, _ = self._process_information_no_sysmgr(
self.device)
# Make sure time has changed
if not self.previous_pid != previous_pid:
temp.failed("The restart pid has changed for "
"process '{p}'".format(p=self.process))
timeout.sleep()
temp.passed('The pid has not changed')
break
def _reconnect(self, steps, timeout):
'''Reconnect to the device if needed'''
if self.process in self.reconnect:
ha = self.abstract.sdk.libs.abstracted_libs.ha.HA(
device=self.device)
with steps.start(
'The device is reloading when restarting this process',
continue_=True) as step:
disconnect_device(self.device)
time.sleep(30)
temp = TempResult(container=step)
while timeout.iterate():
try:
connect_device(self.device)
except Exception as e:
temp.failed('Could not reconnect to the device',
from_exception=e)
timeout.sleep()
continue
temp.passed('Reconnected to the device')
break
temp.result()
# check show module
with steps.start('Check module status after reconnection',
continue_=True) as step:
temp = TempResult(container=step)
while timeout.iterate():
try:
ha.check_module()
except AttributeError as e:
temp.failed(
'Could not find mandatory information from show module',
from_exception=e)
continue
except AssertionError as e:
temp.failed('Modules are not ready', from_exception=e)
timeout.sleep()
continue
temp.passed('Modules are ready')
break
temp.result()
def verify_process(self, repeat_restart, steps, timeout):
'''Verify the process has been restarted
Args:
uut (`obj`): Device object.
steps (`step obj`): aetest step object
Returns:
None
'''
with steps.start('Verify process has restarted correctly') as step:
temp = TempResult(container=step)
while timeout.iterate():
output = self.abstract.parser.show_system.\
ShowSystemInternalSysmgrServiceName(device=\
self.device).parse(process=self.process)
if 'instance' not in output:
temp.failed("No output for 'show system internal sysmgr "
"service name {p}'".format(p=self.process))
timeout.sleep()
continue
# Check the if the process has changed pid
try:
pid = output['instance'][self.instance]['tag'][
self.tag]['pid']
sap = output['instance'][self.instance]['tag'][
self.tag]['sap']
restart_count = output['instance'][self.instance]['tag'][
self.tag]['restart_count']
last_restart = output['instance'][self.instance]['tag'][
self.tag]['last_restart_date']
last_restart_time = datetime.datetime.strptime(
last_restart, '%a %b %d %H:%M:%S %Y')
except Exception as e:
temp.failed("Issue retrieving information about "
"'{p}' process".format(p=self.process),
from_exception=e)
# Make sure time has changed
if not self.last_restart_time < last_restart_time:
temp.failed("The restart time has not changed for "
"process '{p}'".format(p=self.process))
timeout.sleep()
continue
# Make sure the pid has changed
if self.process not in self.reconnect and pid == self.previous_pid:
temp.failed("The pid has not changed for process '{p}'"
"\nprevious pid: {pp}"
"\ncurrent pid: "
"{cp}".format(p=self.process,
pp=self.previous_pid,
cp=pid))
timeout.sleep()
continue
# Verify the restart_count has increased
if self.process not in self.reconnect and\
self.previous_restart_count + repeat_restart != restart_count:
temp.failed('Restart count has not increased by {rr}'
'\nprevious count: {pc}'
'\ncurrent count: {cc}'.format(
rr=repeat_restart,
pc=self.previous_restart_count,
cc=restart_count))
timeout.sleep()
continue
# exclude sap when the value is not in range [0, 1023]
if sap > 1023:
self.verify_exclude.append('sap')
# Modify the original output so it does not fail the compare
self.previous_output['instance'][self.instance]['tag'][self.tag]['restart_count'] =\
restart_count
self.previous_output['instance'][self.instance]['tag'][self.tag]['pid'] =\
pid
diff = Diff(self.previous_output,
output,
exclude=self.verify_exclude)
diff.findDiff()
if diff.diffs:
temp.failed(
"The device output has changed in an unexpected "
"way\n{d}".format(d=str(diff.diffs)))
timeout.sleep()
continue
break
else:
temp.result()
def _reconnect(self, steps, timeout, sleep_disconnect=30):
"""Disconnect and reconnect to router within given timeout.
Args:
Mandatory:
steps (`obj`) : Step object to represent each step taken.
timeout (`obj`) :
max_time (int): Maximum wait time for the trigger,
in second. Default: 180
interval (int): Wait time between iterations when looping is needed,
in second. Default: 15
Optional:
sleep_disconnect (`int`) : Break between issue the command and the
HA action really take place,
in second. Default: 30
Returns:
AETEST Step Result
Raises:
None
Example:
>>> _reconnect(steps=ats.aetest.Steps(),
timeout=genie.utils.timeout.Timeout(
max_time=180,
interval=15))
"""
if hasattr(self.device.custom, 'vdcs'):
devices = [self.device]
for vdc in self.device.custom['vdcs']:
devices.append(self.device.testbed.devices[vdc])
else:
devices = [self.device]
for device in devices:
with steps.start('Disconnecting device {}'.format(device.name),
continue_=True) as step:
disconnect_device(device)
time.sleep(sleep_disconnect)
with steps.start('Reconnecting to device {}'.format(device.name),
continue_=True) as step:
temp = TempResult(container=step)
while timeout.iterate():
try:
connect_device(device)
except Exception as e:
temp.failed('Could not reconnect to the device',
from_exception=e)
# incase console is existed but cannot enable the device.
# conf mode is not active when standby RP is coming up
try:
disconnect_device(device)
except:
pass
timeout.sleep()
continue
temp.passed('Reconnected to the device')
break
temp.result()
def _reconnect(self, steps, timeout):
'''Reconnect to the device if needed'''
if self.process in self.reconnect:
ha = self.abstract.sdk.libs.abstracted_libs.ha.HA(
device=self.device)
with steps.start(
'The device is reloading when restarting this process',
continue_=True) as step:
# Have to sleep for some time for 1 RP to become active
# Pass scenario:
# This supervisor (sup-1)
# -----------------------
# Redundancy state: Active
# Supervisor state: Active
# Internal state: Active with no standby
# Other supervisor (sup-2)
# ------------------------
# Redundancy state: Offline
# Fail scenario:
# This supervisor (sup-1)
# -----------------------
# Redundancy state: Offline
# Supervisor state: Offline
# Internal state: Other
# Other supervisor (sup-2)
# ------------------------
# Redundancy state: Standby
active_wait = Timeout(max_time=50, interval=20)
log.info("Swap consoles standby to active")
self.device.swap_roles()
while active_wait.iterate():
log.info("Checking redundancy offline status")
self.device.execute('show system redundancy status')
active_wait.sleep()
continue
temp = TempResult(container=step)
log.info("\n\nTrying to connect to consoles")
connect_timeout = Timeout(max_time=300, interval=60)
while connect_timeout.iterate():
try:
self.device.connect(prompt_recovery=True)
except Exception as e:
temp.failed('Could not reconnect to the device',
from_exception=e)
connect_timeout.sleep()
continue
temp.passed('Reconnected to the device')
break
temp.result()
# check show module
with steps.start('Check module status after reconnection',
continue_=True) as step:
temp = TempResult(container=step)
module_time = Timeout(max_time=1200, interval=60)
while module_time.iterate():
try:
ha.check_module()
except AttributeError as e:
temp.failed(
'Could not find mandatory information from show module',
from_exception=e)
continue
except AssertionError as e:
temp.failed('Modules are not ready', from_exception=e)
module_time.sleep()
continue
temp.passed('Modules are ready')
break
temp.result()
def _reconnect(self, steps, timeout):
'''Reconnect to the device if needed'''
if self.process in self.reconnect:
ha = self.abstract.sdk.libs.abstracted_libs.ha.HA(
device=self.device)
with steps.start(
'The device is reloading when restarting this process',
continue_=True) as step:
# Waiting 4-7 mins for switchover to complete
log.info(
"Waiting for a maximum of 7 minutes for device switchover to completess"
)
check_time = Timeout(max_time=700, interval=20)
while check_time.iterate():
check_time.sleep()
continue
self.device.destroy()
time.sleep(30)
temp = TempResult(container=step)
while timeout.iterate():
try:
self.device.connect(prompt_recovery=True)
except Exception as e:
temp.failed('Could not reconnect to the device',
from_exception=e)
timeout.sleep()
continue
temp.passed('Reconnected to the device')
break
temp.result()
# check show module
with steps.start('Check module status after reconnection',
continue_=True) as step:
temp = TempResult(container=step)
while timeout.iterate():
try:
ha.check_module()
except AttributeError as e:
temp.failed(
'Could not find mandatory information from show module',
from_exception=e)
continue
except AssertionError as e:
temp.failed('Modules are not ready', from_exception=e)
timeout.sleep()
continue
temp.passed('Modules are ready')
break
temp.result()
