def ae_reconfig(**kwargs): """ :param kwargs: :return: """ t.log("inside ae_reconfig actions") cst._check_bbe() device_id = kwargs.get('device_id', bbe.get_devices(device_tags='dut', id_only=True)[0]) router = t.get_handle(device_id) ae_members = cst.get_ae_info(**kwargs) if not ae_members: t.log("No AE interface exists, quit this testcase") return ae_intf = random.choice(list(ae_members.keys())) port_list = ae_members[ae_intf]['active'] + ae_members[ae_intf]['standby'] chosen_port = random.choice(port_list) router.cli(command="show interfaces {} terse".format(chosen_port)) resp = router.cli( command="show configuration interfaces | display set | match {}". format(chosen_port)).resp command_list = resp.splitlines() command_list.pop(0) command_list.pop(-1) t.log("start reconfig ae {} member link {}".format(ae_intf, chosen_port)) router.config(command_list=['delete interface' + chosen_port]) router.commit() router.cli(command="show interfaces {} terse".format(chosen_port)) time.sleep(10) router.config(command_list=command_list) router.commit() router.cli(command="show interfaces {} terse".format(chosen_port))
def interface_bounce(**kwargs): """ interface_bounce(interface='xe-2/0/0:0', method='cli') :param kwargs: method: bounce method, cli/laseroff , by default id laseroff interface: the interface name interface_id: interface id name , e.g. 'radius0' device_id: device name, by default is 'r0' :return: """ t.log("inside interface bounce action") cst._check_bbe() device = kwargs.get('device_id', bbe.get_devices(device_tags='dut', id_only=True)[0]) router = t.get_handle(device) method = kwargs.get('method', 'laseroff').lower() intfs = bbe.get_interfaces(device=device, interfaces='access') tester = t.get_handle('rt0') status = True interfaces = [] for intf in intfs: interfaces.append(intf.interface_pic) ##if interface was not provided, choose a access interface randomly radius_interface = False if 'interface' in kwargs: interface = kwargs['interface'] elif 'interface_id' in kwargs: interface = bbe.get_interfaces( device=device, interfaces=kwargs['interface_id'])[0].interface_pic if 'radius' in kwargs['interface_id']: radius_interface = True else: interface = random.choice(interfaces) router.cli(command='show interface {} terse'.format(interface)) if method == 'cli': t.log('bounce(disable) interface {}'.format(interface)) if radius_interface: router.cli(command="show network-access aaa radius-servers detail") if 'vcp' in interface: groups = re.match(r'vcp-(\d+)\/(\d+)\/(\d+)', interface) router.cli( command= 'request virtual-chassis vc-port delete fpc-slot {} pic-slot {} port {}' .format(groups.group(1), groups.group(2), groups.group(3))) time.sleep(5) resp = router.cli( command="show interface {}".format(interface)).resp if 'error' in resp: t.log("interface {} was removed for vcps".format(interface)) else: t.log( 'ERROR', "interface {} was not removed for vcps".format(interface)) status = False router.cli( command= 'request virtual-chassis vc-port set fpc-slot {} pic-slot {} port {}' .format(groups.group(1), groups.group(2), groups.group(3))) time.sleep(5) resp = router.cli( command="show interface {}".format(interface)).resp if 'error' in resp: t.log('ERROR', "interface {} was not added for vcps".format(interface)) status = False else: t.log("interface {} was added for vcps".format(interface)) else: for cmds in ['set', 'delete']: if cmds == 'set': expected_state = 'down' if cmds == 'delete': expected_state = 'up' command_list = [ cmds + ' interface ' + interface + ' disable', 'commit' ] resp = router.config(command_list=command_list).resp if 'error' in resp: t.log( 'ERROR', 'the command {} was not accepted by router'.format( command_list)) status = False router.config(command_list=['rollback']) break time.sleep(5) resp = router.pyez('get_interface_information', level_extra='terse', interface_name=interface).resp state = resp.findtext('physical-interface/oper-status') if state != expected_state: t.log( 'ERROR', 'interface {} state is {} after set action {}'.format( interface, state, cmds)) status = False else: t.log( 'interface {} state is {} after set action {}'.format( interface, state, cmds)) if radius_interface: router.cli( command="show network-access aaa radius-servers detail" ) if method == 'laseroff': rt_port_handle = None for intf in intfs: if interface == intf.interface_pic: interface_id = intf.interface_id rt_interface = bbe.get_connection( device, interface=interface_id).interface_pic rt_port_handle = tester.port_to_handle_map[rt_interface] break rt_interface = bbe.get_connection(device, interface) t.log('bounce the tester port {} which connect to {}'.format( rt_interface, interface)) for action in ['laseroff', 'laseron']: if action == 'laseroff': op_mode = 'sim_disconnect' expected_state = 'down' if action == 'laseron': op_mode = 'normal' expected_state = 'up' tester.invoke('interface_config', port_handle=rt_port_handle, op_mode=op_mode) time.sleep(5) resp = router.pyez('get_interface_information', level_extra='terse', interface_name=interface).resp state = resp.findtext('physical-interface/oper-status') if state != expected_state: t.log( 'ERROR', 'interface {} state is {} after set action {}'.format( interface, state, action)) status = False else: t.log('interface {} state is {} after set action {}'.format( interface, state, action)) if not status: raise Exception( 'interface {} state not in expected state after flapping'.format( interface))
def fpc_actions(**kwargs): """ fpc_actions(fpc='1', action='offon') :param kwargs: fpc: fpc slot id e.g. '1' device_id: router id e.g. 'r0' action: restart/panic/offon :return: """ t.log("inside fpc_actions") cst._check_bbe() device_id = kwargs.get('device_id', bbe.get_devices(device_tags='dut', id_only=True)[0]) router = t.get_handle(device_id) fpc_list = [] if 'fpc' in kwargs: chosen_fpc = kwargs['fpc'] else: access_list = bbe.get_interfaces(device=device_id, interfaces='access') if not access_list: t.log('WARN', "no access interface exist in yaml, skip the test") return for access in access_list: match = re.match(r'.*-(\d+)/\d+/\d+', access.interface_pic) if match: fpc_list.append(match.group(1)) chosen_fpc = random.choice(fpc_list) action_list = ['restart', 'panic', 'offon'] if 'action' in kwargs: action = kwargs['action'] else: action = random.choice(action_list) t.log("start fpc {} with action {}".format(chosen_fpc, action)) if action == 'restart': t.log('will restart fpc slot {}'.format(chosen_fpc)) command = 'request chassis fpc restart slot {}'.format(chosen_fpc) resp = router.cli(command=command).resp match = re.match(r'Restart\s+initiated', resp) if match: t.log('fpc slot #{} restarted'.format(chosen_fpc)) else: raise Exception( 'fpc slot {} can not be restarted'.format(chosen_fpc)) if action == 'panic': t.log('will panic fpc slot {}'.format(chosen_fpc)) resp = router.vty(command='set parser security 10', destination='fpc' + chosen_fpc).resp if re.search('Security level', resp): t.log('enter into vty security mode') router.vty(command='test panic', destination='fpc' + chosen_fpc, pattern='(.*)') t.log("waiting for core-dumps to be generated") # Wait for the core to be generated core_retries = 100 core_interval = 5 fpc_core_found = False while core_retries > 0: resp = router.cli(command='show system core-dumps').resp if not re.search('core-', resp): core_retries -= 1 else: fpc_core_found = True t.log("core dump generated for fpc {}".format(chosen_fpc)) break time.sleep(core_interval) if fpc_core_found is True: t.log( 'fpc core was generated during test panic, will remove it') router.cli(command='file delete /var/crash/core-*') else: raise Exception("not able to set vty security mode") if action == 'offon': t.log("will offline/online fpc slot {}".format(chosen_fpc)) for item in ['offline', 'online']: if not re.match(r'MX(80|80-t|40|40-t|10|10-t|5|5-t)', router.get_model(), re.IGNORECASE): command = "request chassis fpc slot {} {}".format( chosen_fpc, item) else: command = "request chassis tfeb {}".format(item) resp = router.cli(command=command).resp if item == 'offline': match = re.match(r'Offline\s+initiated', resp) if match: t.log('fpc slot #{} offline'.format(chosen_fpc)) else: raise Exception( 'fpc slot {} failed to be offline'.format(chosen_fpc)) else: t.log('fpc slot {} enabled online command'.format(chosen_fpc)) t.log("waiting 100s for fpc to come back") time.sleep(100) base_time = time.time() while True: resp = router.pyez('get_fpc_information', fpc_slot=chosen_fpc).resp if resp.findtext('fpc/state') == 'Online': t.log('fpc slot {} back to online'.format(chosen_fpc)) break else: time.sleep(10) if (time.time() - base_time) > 600: raise Exception( "FPC slot {} failed to come back online after 600s".format( chosen_fpc)) rt_count = cst.get_rt_subs_info()['rt_sessions_up'] router_count = cst.get_router_sub_summary(device_id)['active'] time.sleep(10) base_time = time.time() while True: new_rt_count = cst.get_rt_subs_info()['rt_sessions_up'] new_router_count = cst.get_router_sub_summary(device_id)['active'] if new_router_count == router_count and new_rt_count == rt_count: t.log("router and tester user reached stable state") break else: rt_count = new_rt_count router_count = new_router_count t.log("Waiting 60s for client count to be stable on dut and rt") time.sleep(60) if time.time() - base_time > 3600: raise Exception("subscribers count still not stable after 3600s")
def daemon_actions(**kwargs): """ daemon_actions(daemon_list=['jdhcpd','bbe-smgd'], re_in_action='re1',action='kill') :param kwargs: device_id: device name e.g. 'r0' daemon_list: a list of daemons action: restart/coredump/kill, by default is restart re_in_action: re name, e.g. 're0' cpu_settle: cpu idle value, by default is 50 cpu_check: True/False, default is True verify_traffic: True/False, by default is False traffic_args: :return: """ t.log("inside daemon_actions") cst._check_bbe() verify_traffic_enable = kwargs.get('verify_traffic', False) cpu_check = kwargs.get('cpu_check', True) device_id = kwargs.get('device_id', bbe.get_devices(device_tags='dut', id_only=True)[0]) rt_device_id = kwargs.get('rt_device_id', 'rt0') tester = t.get_handle(rt_device_id) subs = kwargs.get('subs', bbe.get_subscriber_handles()) if verify_utils.convert_str_to_num_or_bool(verify_traffic_enable): t.log('start login subs {} before test'.format(subs)) try: cst.cst_start_clients(**kwargs) except: t.log("failed to bring up subscriber") if verify_utils.convert_str_to_num_or_bool(verify_traffic_enable): new_mesh = kwargs.get('new_mesh', True) duration = kwargs.get('duration', 60) if new_mesh: t.log('remove existing traffic mesh') tester.invoke('traffic_action', action='delete') time.sleep(5) t.log('add new traffic mesh for subs') cst.add_subscriber_mesh(**kwargs) if 'duration' in kwargs: cst.start_traffic(**kwargs) else: cst.start_traffic(duration=duration, **kwargs) time.sleep(int(duration)) cst.verify_traffic(**kwargs) cst.start_traffic() daemon_action_cmds = {} daemon_action_cmds['authd'] = { 'restart_cmd': 'restart general-authentication-service', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['jdhcpd'] = { 'restart_cmd': 'restart dhcp-service', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['dcd'] = { 'restart_cmd': 'restart interface-control', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['dfwd'] = { 'restart_cmd': 'restart firewall', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['cosd'] = { 'restart_cmd': 'restart class-of-service', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['pppd'] = { 'restart_cmd': 'restart ppp', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['jpppd'] = { 'restart_cmd': 'restart ppp-service', 'cmds_before_action': [ "show route | grep Access-internal | count", "show route terse | grep pp0 | count", "show route | grep demux | count" ], 'cmds_after_action': [ "show route | grep Access-internal | count", "show route terse | grep pp0 | count", "show route | grep demux | count" ] } daemon_action_cmds['pppoed'] = { 'restart_cmd': 'restart pppoe', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['l2-learning'] = { 'restart_cmd': 'restart l2-learning', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['l2cpd-service'] = { 'restart_cmd': 'restart l2cpd-service', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['jl2tpd'] = { 'restart_cmd': 'restart l2tp-universal-edge', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['dfcd'] = { 'restart_cmd': 'restart dynamic-flow-capture', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['autoconfd'] = { 'restart_cmd': 'restart auto-configuration', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['bbe-smgd'] = { 'restart_cmd': 'restart smg-service', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['pfed'] = { 'restart_cmd': 'restart statistics-service', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['smid'] = { 'restart_cmd': 'restart subscriber-management', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['bdbrepd'] = { 'restart_cmd': 'restart database-replication', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['mib2d'] = { 'restart_cmd': 'restart mib-process', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['snmpd'] = { 'restart_cmd': 'restart snmp', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['eventd'] = { 'restart_cmd': 'restart event-processing', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['essmd'] = { 'restart_cmd': 'restart extensible-subscriber-services', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['rpd'] = { 'restart_cmd': 'restart routing immediately', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['diameterd'] = { 'restart_cmd': 'restart diameter-service immediately', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['lacpd'] = { 'restart_cmd': 'restart lacp', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['pppoed'] = { 'restart_cmd': 'restart pppoe', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['chassisd'] = { 'restart_cmd': 'restart chassis-control', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['ancpd'] = { 'restart_cmd': 'restart ancpd-service', 'cmds_before_action': [], 'cmds_after_action': [] } daemon_action_cmds['bbe-statsd'] = { 'restart_cmd': 'restart bbe-stats-service', 'cmds_before_action': [], 'cmds_after_action': [] } if 'nfx' in t.get_system(device_id)['primary']['model']: daemon_action_cmds['chassisd'][ 'restart_cmd'] = 'restart chassis-control soft' if 'mx204' in t.get_system(device_id)['primary']['model']: daemon_action_cmds['chassisd'][ 'restart_cmd'] = 'restart chassis-control soft' default_list = list(daemon_action_cmds.keys()) daemon_list = kwargs.get('daemon_list', default_list) if not isinstance(daemon_list, list): daemon_list = [daemon_list] device = kwargs.get('device_id', bbe.get_devices(device_tags='dut', id_only=True)[0]) router = t.get_handle(device) default_cmds_before_action = [ 'show subscribers summary', 'show subscribers summary port' ] default_cmds_after_action = default_cmds_before_action master_re_name = cst.get_master_re_name(device) re_name = kwargs.get('re_in_action', master_re_name) if re_name != master_re_name: if hasattr(router, 'vc') and router.vc: master_node = re_name.split(sep='-')[0] rename = re_name.split(sep='-')[1] router.set_current_controller(controller=rename, system_node=master_node) else: router.set_current_controller(controller=re_name, system_node='primary') t.log('will start action in re {}'.format(re_name)) action = kwargs.get('action', 'restart') for daemon in daemon_list: if daemon not in daemon_action_cmds: t.log( "WARN", "daemon {} is not defined in daemon_action_cmds, will be skipped" .format(daemon)) continue resp = router.shell(command='ps -aux |grep {}'.format(daemon)).resp if not re.search('/' + daemon, resp): t.log("daemon {} is not running, will be skipped from the test". format(daemon)) continue cmds_before_action = daemon_action_cmds[daemon][ 'cmds_before_action'] or default_cmds_before_action for cmds in cmds_before_action: router.cli(command=cmds, timeout=300) time.sleep(5) if action == 'restart': t.log("daemon {} will be restarted".format(daemon)) router.cli(command=daemon_action_cmds[daemon]['restart_cmd']) if action in ['coredump', 'kill']: command = "ps axf | grep {} | grep -v grep".format(daemon) resp = router.shell(command=command).resp if re.match(r'\s?(\d+)\s+', resp): pid = re.match(r'\s?(\d+)\s+', resp).group(1) else: t.log("daemon {} is not in the system, will ignore it".format( daemon)) continue router.su() if action == 'coredump': # Delete any daemon core if exists router.cli( command='file delete /var/tmp/{}.core*'.format(daemon)) # Kill daemon with SIGABRT router.shell(command="kill -6 " + pid) # Check daemon is restarted with new pid restarted_with_new_pid = False new_pid_retries = 25 new_pid_interval = 5 new_pid = pid while new_pid_retries > 0: time.sleep(new_pid_interval) resp = router.shell(command=command).resp m = re.match(r'\s?(\d+)\s+', resp) if m is not None: new_pid = m.group(1) else: t.log("daemon {} is not started yet".format(daemon)) if m is not None and new_pid != pid: restarted_with_new_pid = True t.log("daemon {} restarted with new pid {}".format( daemon, new_pid)) break new_pid_retries -= 1 if restarted_with_new_pid is False: t.log("{} is not restarted after {} seconds".format( daemon, new_pid_retries * new_pid_interval)) raise Exception("daemon was not restarted") # Check daemon core dump core_retries = 100 core_interval = 5 daemon_core_found = False while core_retries > 0: resp = router.shell( command="ls /var/tmp/{}.core*".format(daemon)).resp if not re.search(daemon, resp): core_retries -= 1 else: daemon_core_found = True t.log("core dump was generated for daemon {}".format( daemon)) break time.sleep(core_interval) if daemon_core_found is False: raise Exception( "no core was generated for daemon {}".format(daemon)) # Delete generated daemon core file # ywang tested on 19.2 that when daemon is core dumping, the helper is dumpd. dump_completed = False dump_retries = 100 dump_interval = 10 dump_cmd = "ps axf | grep dumpd | grep -v grep" while dump_retries > 0: resp = router.shell(command=dump_cmd).resp if re.search('dumpd', resp, re.MULTILINE): time.sleep(dump_interval) dump_retries -= 1 continue else: dump_completed = True break if dump_completed: router.cli( command='file delete /var/tmp/{}.core*'.format(daemon)) t.log("Deleted core dump generated by {}".format(daemon)) else: t.log("{} core dump is not completed after {} seconds". format(daemon, dump_retries * dump_interval)) raise Exception("daemon core dump takes too long") if action == 'kill': router.shell(command="kill -9 " + pid) time.sleep(5) resp = router.shell(command=command).resp new_pid = re.match(r'\s?(\d+)\s+', resp).group(1) if new_pid == pid: #router.set_current_controller(controller=master_re_name, system_node='primary') raise Exception("daemon {} was not killed".format(daemon)) else: t.log("daemon {} was killed, new pid is {}".format( daemon, new_pid)) if cpu_check: BBEJunosUtil.cpu_settle( cpu_threshold=int(kwargs.get('cpu_process', 30)), idle_min=int(kwargs.get('cpu_settle', '75')), dead_time=int(kwargs.get('cpu_deadtime', 1200)), interval=int(kwargs.get('cpu_interval', 20))) check_client = kwargs.get('verify_client', False) if verify_utils.convert_str_to_num_or_bool(check_client): cst.verify_client_count(**kwargs) cmds_after_action = default_cmds_after_action if not daemon_action_cmds[daemon]['cmds_after_action'] else \ daemon_action_cmds[daemon]['cmds_after_action'] for cmds in cmds_after_action: router.cli(command=cmds, timeout=300) if verify_utils.convert_str_to_num_or_bool(verify_traffic_enable): cst.stop_traffic() cst.verify_traffic(**kwargs) if daemon != daemon_list[-1]: cst.start_traffic() if re_name != master_re_name: router.set_current_controller(controller=master_re_name, system_node='primary') t.log('switch back to master re {}'.format(master_re_name))
def re_actions(**kwargs): """ re_actions(type='backup', action='kernel_crash') used to power cycle RE, default for backup re power cycle :param kwargs: device_id: device id, default is 'r0' type: master/backup, default is backup action: reboot/power_cycle/kernel_crash, default is reboot :return: """ t.log("inside re_actions") cst._check_bbe() device_id = kwargs.get('device_id', bbe.get_devices(device_tags='dut', id_only=True)[0]) router = t.get_handle(device_id) re_type = kwargs.get('type', 'backup') action = kwargs.get('action', 'reboot') origin_re = router.get_current_controller_name() backup_re = None for re_name in t.get_resource( device_id)['system']['primary']['controllers']: if re_name == router.get_current_controller_name(): continue backup_re = re_name break picked_re = backup_re if re_type == 'backup' else origin_re if re_type == 'master': router.set_current_controller(controller=backup_re, system_node='primary') match = re.match(r're(\d+)', picked_re) if match: re_slot = match.group(1) if action == 'power_cycle': for command in ['power-off', 'power-on']: router.cli(command="show subscribers summary") t.log("RE {} will start powercycle".format(picked_re)) router.cli(command="request system {} other-routing-engine".format( command)) t.log("waiting 60s for route-engine state change") time.sleep(60) router.cli(command="show subscribers summary") if command == 'power-off': resp = router.pyez('get_route_engine_information', slot=re_slot).resp if resp.findtext('route-engine/mastership-state') == 'Present': t.log('routing-engine {} was {}'.format( picked_re, command)) else: raise Exception( "routing-engine {} was in abnormal state".format( picked_re)) if command == 'power-on': t.log("routing-engine {} was {}".format(picked_re, command)) cst.check_re_status(device_id=device_id, re_slot=re_slot) router.cli(command="show subscribers summary") if action == 'reboot': router.cli(command="show subscribers summary") t.log("RE {} will start reboot".format(picked_re)) router.cli(command="request system reboot other-routing-engine") t.log("waiting 60s for route-engine state change") time.sleep(60) router.cli(command="show subscribers summary") cst.check_re_status(device_id=device_id, re_slot=re_slot) router.cli(command="show subscribers summary") if action == 'kernel_crash': router.cli(command="show subscribers summary") bbe_router = bbe.get_devices(devices=device_id)[0] if bbe_router.is_tomcat: tomcat_mode = True host = t.get_resource(device_id)['system']['primary']['controllers'][picked_re]['con-ip'] + '.' + \ t.get_resource(device_id)['system']['primary']['controllers'][picked_re]['domain'] t.log("will panic re {}".format(picked_re)) cst.panic_re_recover(host=host, tomcat_mode=tomcat_mode) t.log("waiting 60s for route-engine state change") time.sleep(60) cst.check_re_status(device_id=device_id, re_slot=re_slot) router.cli(command="show subscribers summary") router.reconnect(all)