def _verify_restart_no_sysmgr(self, repeat_restart, steps, timeout): '''Verify with show process that the pid has changed''' # TODO # Verify if timestamps is increasing # TO BE TESTED WHEN SWITCH TO UNICON # with steps.start('Verify the timestamp "previous < current"', # continue_=True) as step: # # get timestamp # timestamp = self.get_process_timestamp(process=self.process) # if timestamp <= self.previous_timestamp: # temp.failed("The timestamp is not increasing after restart " # "process '{p}'".format(p=self.process)) # else: # temp.passed('The timestamp incresaed') # Verify if the pid is not the same with steps.start('Verify if the pid is the same', continue_=True) as step: temp = TempResult(container=step) while timeout.iterate(): previous_pid, _ = self._process_information_no_sysmgr() # Make sure time has changed if not self.previous_pid != previous_pid: temp.failed("The restart pid has changed for " "process '{p}'".format(p=self.process)) timeout.sleep() temp.passed('The pid has not changed') break
def _extra_core(self, steps): with steps.start('Verify if no extra core has been generated', continue_=True) as step: # Check if core of process is found filetransfer = self.device.filetransfer if \ hasattr(self.device, 'filetransfer') else None ha = self.abstract.sdk.libs.abstracted_libs.ha.HA( device=self.device, filetransfer=filetransfer) temp = TempResult(container=step) cores = ha.check_cores() for core in cores: # Upload core of all the other core found if int(core['pid']) == self.previous_pid: continue # Different core than the one expected! log.error("Process '{p}' with '{pp}' has created a core "\ "but should not have".\ format(p=core['process'], pp=core['pid'])) temp.failed('More core than expected has been created') try: ha.upload_core_to_linux(core) except Exception as e: log.error('Cannot upload core files\n{}'.format(str(e))) # Clear cores ha.clear_cores() temp.result()
def _reconnect(self, steps, timeout, sleep_disconnect=30): """Disconnect and reconnect to router within given timeout. Args: Mandatory: steps (`obj`) : Step object to represent each step taken. timeout (`obj`) : max_time (int): Maximum wait time for the trigger, in second. Default: 180 interval (int): Wait time between iterations when looping is needed, in second. Default: 15 Optional: sleep_disconnect (`int`) : Break between issue the command and the HA action really take place, in second. Default: 30 Returns: AETEST Step Result Raises: None Example: >>> _reconnect(steps=ats.aetest.Steps(), timeout=genie.utils.timeout.Timeout( max_time=180, interval=15)) """ with steps.start('Disconnecting device {}'.format(self.device.name), continue_=True) as step: disconnect_device(self.device) time.sleep(sleep_disconnect) with steps.start('Reconnecting to device {}'.format(self.device.name), continue_=True) as step: temp = TempResult(container=step) while timeout.iterate(): try: connect_device(self.device) except Exception as e: temp.failed('Could not reconnect to the device', from_exception=e) # incase console is existed but cannot enable the device. # conf mode is not active when standby RP is coming up try: disconnect_device(self.device) except: pass timeout.sleep() continue temp.passed('Reconnected to the device') break temp.result()
def _reconnect(self, steps, timeout): '''Reconnect to the device if needed''' if self.process in self.reconnect: ha = self.abstract.sdk.libs.abstracted_libs.ha.HA( device=self.device) with steps.start( 'The device is reloading when restarting this process', continue_=True) as step: # Waiting 4-7 mins for switchover to complete log.info( "Waiting for a maximum of 7 minutes for device switchover to completess" ) check_time = Timeout(max_time=700, interval=20) while check_time.iterate(): check_time.sleep() continue self.device.destroy() time.sleep(30) temp = TempResult(container=step) while timeout.iterate(): try: self.device.connect(prompt_recovery=True) except Exception as e: temp.failed('Could not reconnect to the device', from_exception=e) timeout.sleep() continue temp.passed('Reconnected to the device') break temp.result() # check show module with steps.start('Check module status after reconnection', continue_=True) as step: temp = TempResult(container=step) while timeout.iterate(): try: ha.check_module() except AttributeError as e: temp.failed( 'Could not find mandatory information from show module', from_exception=e) continue except AssertionError as e: temp.failed('Modules are not ready', from_exception=e) timeout.sleep() continue temp.passed('Modules are ready') break temp.result()
def _reconnect(self, steps, timeout): '''Reconnect to the device if needed''' if self.process in self.reconnect: ha = self.abstract.sdk.libs.abstracted_libs.ha.HA( device=self.device) with steps.start( 'The device is reloading when restarting this process', continue_=True) as step: disconnect_device(self.device) time.sleep(30) temp = TempResult(container=step) while timeout.iterate(): try: connect_device(self.device) except Exception as e: temp.failed('Could not reconnect to the device', from_exception=e) timeout.sleep() continue temp.passed('Reconnected to the device') break temp.result() # check show module with steps.start('Check module status after reconnection', continue_=True) as step: temp = TempResult(container=step) while timeout.iterate(): try: ha.check_module() except AttributeError as e: temp.failed( 'Could not find mandatory information from show module', from_exception=e) continue except AssertionError as e: temp.failed('Modules are not ready', from_exception=e) timeout.sleep() continue temp.passed('Modules are ready') break temp.result()
def verify_process(self, repeat_restart, steps, timeout): '''Verify the process has been restarted Args: uut (`obj`): Device object. steps (`step obj`): aetest step object Returns: None ''' with steps.start('Verify process has restarted correctly') as step: temp = TempResult(container=step) while timeout.iterate(): output = self.abstract.parser.show_system.\ ShowSystemInternalSysmgrServiceName(device=\ self.device).parse(process=self.process) if 'instance' not in output: temp.failed("No output for 'show system internal sysmgr " "service name {p}'".format(p=self.process)) timeout.sleep() continue # Check the if the process has changed pid try: pid = output['instance'][self.instance]['tag'][ self.tag]['pid'] sap = output['instance'][self.instance]['tag'][ self.tag]['sap'] restart_count = output['instance'][self.instance]['tag'][ self.tag]['restart_count'] last_restart = output['instance'][self.instance]['tag'][ self.tag]['last_restart_date'] last_restart_time = datetime.datetime.strptime( last_restart, '%a %b %d %H:%M:%S %Y') except Exception as e: temp.failed("Issue retrieving information about " "'{p}' process".format(p=self.process), from_exception=e) # Make sure time has changed if not self.last_restart_time < last_restart_time: temp.failed("The restart time has not changed for " "process '{p}'".format(p=self.process)) timeout.sleep() continue # Make sure the pid has changed if self.process not in self.reconnect and pid == self.previous_pid: temp.failed("The pid has not changed for process '{p}'" "\nprevious pid: {pp}" "\ncurrent pid: " "{cp}".format(p=self.process, pp=self.previous_pid, cp=pid)) timeout.sleep() continue # Verify the restart_count has increased if self.process not in self.reconnect and\ self.previous_restart_count + repeat_restart != restart_count: temp.failed('Restart count has not increased by {rr}' '\nprevious count: {pc}' '\ncurrent count: {cc}'.format( rr=repeat_restart, pc=self.previous_restart_count, cc=restart_count)) timeout.sleep() continue # exclude sap when the value is not in range [0, 1023] if sap > 1023: self.verify_exclude.append('sap') # Modify the original output so it does not fail the compare self.previous_output['instance'][self.instance]['tag'][self.tag]['restart_count'] =\ restart_count self.previous_output['instance'][self.instance]['tag'][self.tag]['pid'] =\ pid diff = Diff(self.previous_output, output, exclude=self.verify_exclude) diff.findDiff() if diff.diffs: temp.failed( "The device output has changed in an unexpected " "way\n{d}".format(d=str(diff.diffs))) timeout.sleep() continue break else: temp.result()
def _reconnect(self, steps, timeout): '''Reconnect to the device if needed''' if self.process in self.reconnect: ha = self.abstract.sdk.libs.abstracted_libs.ha.HA( device=self.device) with steps.start( 'The device is reloading when restarting this process', continue_=True) as step: # Have to sleep for some time for 1 RP to become active # Pass scenario: # This supervisor (sup-1) # ----------------------- # Redundancy state: Active # Supervisor state: Active # Internal state: Active with no standby # Other supervisor (sup-2) # ------------------------ # Redundancy state: Offline # Fail scenario: # This supervisor (sup-1) # ----------------------- # Redundancy state: Offline # Supervisor state: Offline # Internal state: Other # Other supervisor (sup-2) # ------------------------ # Redundancy state: Standby active_wait = Timeout(max_time=50, interval=20) log.info("Swap consoles standby to active") self.device.swap_roles() while active_wait.iterate(): log.info("Checking redundancy offline status") self.device.execute('show system redundancy status') active_wait.sleep() continue temp = TempResult(container=step) log.info("\n\nTrying to connect to consoles") connect_timeout = Timeout(max_time=300, interval=60) while connect_timeout.iterate(): try: self.device.connect(prompt_recovery=True) except Exception as e: temp.failed('Could not reconnect to the device', from_exception=e) connect_timeout.sleep() continue temp.passed('Reconnected to the device') break temp.result() # check show module with steps.start('Check module status after reconnection', continue_=True) as step: temp = TempResult(container=step) module_time = Timeout(max_time=1200, interval=60) while module_time.iterate(): try: ha.check_module() except AttributeError as e: temp.failed( 'Could not find mandatory information from show module', from_exception=e) continue except AssertionError as e: temp.failed('Modules are not ready', from_exception=e) module_time.sleep() continue temp.passed('Modules are ready') break temp.result()