def _verify_restart_no_sysmgr(self, repeat_restart, steps, timeout):
'''Verify with show process that the pid has changed'''
# TODO
# Verify if timestamps is increasing
# TO BE TESTED WHEN SWITCH TO UNICON
# with steps.start('Verify the timestamp "previous < current"',
# continue_=True) as step:
# # get timestamp
# timestamp = self.get_process_timestamp(process=self.process)
# if timestamp <= self.previous_timestamp:
# temp.failed("The timestamp is not increasing after restart "
# "process '{p}'".format(p=self.process))
# else:
# temp.passed('The timestamp incresaed')
# Verify if the pid is not the same
with steps.start('Verify if the pid is the same',
continue_=True) as step:
temp = TempResult(container=step)
while timeout.iterate():
previous_pid, _ = self._process_information_no_sysmgr()
# Make sure time has changed
if not self.previous_pid != previous_pid:
temp.failed("The restart pid has changed for "
"process '{p}'".format(p=self.process))
timeout.sleep()
temp.passed('The pid has not changed')
break
def _reconnect(self, steps, timeout):
'''Reconnect to the device if needed'''
if self.process in self.reconnect:
ha = self.abstract.sdk.libs.abstracted_libs.ha.HA(
device=self.device)
with steps.start(
'The device is reloading when restarting this process',
continue_=True) as step:
# Waiting 4-7 mins for switchover to complete
log.info(
"Waiting for a maximum of 7 minutes for device switchover to completess"
)
check_time = Timeout(max_time=700, interval=20)
while check_time.iterate():
check_time.sleep()
continue
self.device.destroy()
time.sleep(30)
temp = TempResult(container=step)
while timeout.iterate():
try:
self.device.connect(prompt_recovery=True)
except Exception as e:
temp.failed('Could not reconnect to the device',
from_exception=e)
timeout.sleep()
continue
temp.passed('Reconnected to the device')
break
temp.result()
# check show module
with steps.start('Check module status after reconnection',
continue_=True) as step:
temp = TempResult(container=step)
while timeout.iterate():
try:
ha.check_module()
except AttributeError as e:
temp.failed(
'Could not find mandatory information from show module',
from_exception=e)
continue
except AssertionError as e:
temp.failed('Modules are not ready', from_exception=e)
timeout.sleep()
continue
temp.passed('Modules are ready')
break
temp.result()
def _reconnect(self, steps, timeout, sleep_disconnect=30):
"""Disconnect and reconnect to router within given timeout.
Args:
Mandatory:
steps (`obj`) : Step object to represent each step taken.
timeout (`obj`) :
max_time (int): Maximum wait time for the trigger,
in second. Default: 180
interval (int): Wait time between iterations when looping is needed,
in second. Default: 15
Optional:
sleep_disconnect (`int`) : Break between issue the command and the
HA action really take place,
in second. Default: 30
Returns:
AETEST Step Result
Raises:
None
Example:
>>> _reconnect(steps=ats.aetest.Steps(),
timeout=genie.utils.timeout.Timeout(
max_time=180,
interval=15))
"""
with steps.start('Disconnecting device {}'.format(self.device.name),
continue_=True) as step:
disconnect_device(self.device)
time.sleep(sleep_disconnect)
with steps.start('Reconnecting to device {}'.format(self.device.name),
continue_=True) as step:
temp = TempResult(container=step)
while timeout.iterate():
try:
connect_device(self.device)
except Exception as e:
temp.failed('Could not reconnect to the device',
from_exception=e)
# incase console is existed but cannot enable the device.
# conf mode is not active when standby RP is coming up
try:
disconnect_device(self.device)
except:
pass
timeout.sleep()
continue
temp.passed('Reconnected to the device')
break
temp.result()
def _reconnect(self, steps, timeout):
'''Reconnect to the device if needed'''
if self.process in self.reconnect:
ha = self.abstract.sdk.libs.abstracted_libs.ha.HA(
device=self.device)
with steps.start(
'The device is reloading when restarting this process',
continue_=True) as step:
disconnect_device(self.device)
time.sleep(30)
temp = TempResult(container=step)
while timeout.iterate():
try:
connect_device(self.device)
except Exception as e:
temp.failed('Could not reconnect to the device',
from_exception=e)
timeout.sleep()
continue
temp.passed('Reconnected to the device')
break
temp.result()
# check show module
with steps.start('Check module status after reconnection',
continue_=True) as step:
temp = TempResult(container=step)
while timeout.iterate():
try:
ha.check_module()
except AttributeError as e:
temp.failed(
'Could not find mandatory information from show module',
from_exception=e)
continue
except AssertionError as e:
temp.failed('Modules are not ready', from_exception=e)
timeout.sleep()
continue
temp.passed('Modules are ready')
break
temp.result()
def _reconnect(self, steps, timeout):
'''Reconnect to the device if needed'''
if self.process in self.reconnect:
ha = self.abstract.sdk.libs.abstracted_libs.ha.HA(
device=self.device)
with steps.start(
'The device is reloading when restarting this process',
continue_=True) as step:
# Have to sleep for some time for 1 RP to become active
# Pass scenario:
# This supervisor (sup-1)
# -----------------------
# Redundancy state: Active
# Supervisor state: Active
# Internal state: Active with no standby
# Other supervisor (sup-2)
# ------------------------
# Redundancy state: Offline
# Fail scenario:
# This supervisor (sup-1)
# -----------------------
# Redundancy state: Offline
# Supervisor state: Offline
# Internal state: Other
# Other supervisor (sup-2)
# ------------------------
# Redundancy state: Standby
active_wait = Timeout(max_time=50, interval=20)
log.info("Swap consoles standby to active")
self.device.swap_roles()
while active_wait.iterate():
log.info("Checking redundancy offline status")
self.device.execute('show system redundancy status')
active_wait.sleep()
continue
temp = TempResult(container=step)
log.info("\n\nTrying to connect to consoles")
connect_timeout = Timeout(max_time=300, interval=60)
while connect_timeout.iterate():
try:
self.device.connect(prompt_recovery=True)
except Exception as e:
temp.failed('Could not reconnect to the device',
from_exception=e)
connect_timeout.sleep()
continue
temp.passed('Reconnected to the device')
break
temp.result()
# check show module
with steps.start('Check module status after reconnection',
continue_=True) as step:
temp = TempResult(container=step)
module_time = Timeout(max_time=1200, interval=60)
while module_time.iterate():
try:
ha.check_module()
except AttributeError as e:
temp.failed(
'Could not find mandatory information from show module',
from_exception=e)
continue
except AssertionError as e:
temp.failed('Modules are not ready', from_exception=e)
module_time.sleep()
continue
temp.passed('Modules are ready')
break
temp.result()
