def dtcp_trigger_test(**kwargs):
"""
:param kwargs:
device_id: router device id e.g."r0"
max_trigger_count: maximum trigger count, by default is 100, router support maximum 1024
trigger_type: interface_id/username, by default is interface_id
minimum_rx_percentage: minimum percentage for received traffic
maximum_rx_percentage: maximum percentage for received traffic
:return:
"""
kwargs['mode'] = 'summary'
duration = kwargs.get('duration', 60)
cst.prepare_subscriber_traffic(**kwargs)
max_count = int(kwargs.get('max_trigger_count', 100))
trigger_type = kwargs.get('trigger_type', 'interface_id')
###remove existing triggers
t.log("trying to remove existing LI triggers")
result = dtcp_list_li_trigger(**kwargs, noraise=True)
if isinstance(result, list):
dtcp_delete_li_trigger(criteria_id=result, **kwargs)
t.log("find out the current li candidates")
response = get_dtcp_li_candidates(**kwargs)
if len(response[trigger_type]) < max_count:
max_count = len(response[trigger_type])
users = response[trigger_type]
picked_triggers = random.sample(users, max_count)
t.log("the picked triggers are {}".format(picked_triggers))
t.log("add triggers")
dtcp_add_li_trigger(trigger_list=users, **kwargs)
t.log("verify triggers count")
resp = dtcp_list_li_trigger(**kwargs)
if isinstance(resp, list):
li_count = len(resp)
if li_count == max_count:
t.log(
"all specified {} li triggers in the router".format(li_count))
else:
raise Exception(
"only {} li triggers in the router, not the expected {}".
format(li_count, max_count))
else:
raise Exception("no criteria id returned, please check logs")
if 'duration' in kwargs:
cst.start_traffic(**kwargs)
else:
cst.start_traffic(duration=duration, **kwargs)
time.sleep(int(duration))
maximum_percent = kwargs.get('maximum_rx_percentage', 100.5)
minimum_percent = kwargs.get('minimum_rx_percentage', 99)
t.log("expecting received traffic will be {} - {} percent in LI".format(
minimum_percent, maximum_percent))
cst.verify_traffic(**kwargs)
t.log("remove triggers after test")
dtcp_delete_li_trigger(**kwargs)
def dtcp_concurrent_trigger_test(**kwargs):
"""
:param kwargs:
device_id: router device id e.g."r0"
max_trigger_count: maximum trigger count, by default is 100, router support maximum 1024
minimum_rx_percentage: minimum percentage for received traffic
maximum_rx_percentage: maximum percentage for received traffic
:return:
"""
t.log("start dtcp concurrent trigger test")
max_count = int(kwargs.get('max_trigger_count', 100))
trigger_type = kwargs.get('trigger_type', 'interface_id')
check_traffic = kwargs.get('verify_traffic', True)
delay = kwargs.get('delay', 10)
kwargs['mode'] = 'summary'
cst.prepare_subscriber_traffic(**kwargs)
t.log("trying to remove existing LI triggers")
result = dtcp_list_li_trigger(**kwargs, noraise=True)
if isinstance(result, list):
dtcp_delete_li_trigger(criteria_id=result, **kwargs)
response = get_dtcp_li_candidates(**kwargs)
if len(response[trigger_type]) < max_count:
max_count = len(response[trigger_type])
users = response[trigger_type]
group_a_count = round(len(users) / 2)
group_a = random.sample(users, group_a_count)
t.log("adding the dtcp trigger group first")
dtcp_add_li_trigger(trigger_list=group_a, **kwargs)
result = dtcp_list_li_trigger(**kwargs)
obj1 = lambda: None
obj2 = lambda: None
obj1.member = group_a
obj1.role = 'delete'
obj1.criteria = result
obj1.active = True
group_b = [subs for subs in users if subs not in group_a]
obj2.member = group_b
obj2.role = 'add'
obj2.active = False
obj2.criteria = []
for iteration in range(1, int(kwargs.get('iteration', 2)) + 1):
t.log("start iteration #{} for parallel dtcp test".format(iteration))
for subscriber in [obj1, obj2]:
if subscriber.role == 'delete' and subscriber.active:
dict2 = {
'target': dtcp_delete_li_trigger,
'delay': delay,
'kwargs': {
'criteria_id': subscriber.criteria,
**kwargs
}
}
elif subscriber.role == 'add' and not subscriber.active:
dict1 = {
'target': dtcp_add_li_trigger,
'delay': delay,
'kwargs': {
'trigger_list': subscriber.member,
'trigger_type': trigger_type,
**kwargs
}
}
try:
result = run_multiple([dict1, dict2])
except:
raise Exception("the parallel run of dtcp failed")
if check_traffic:
t.log("verify traffic in iteration {}".format(iteration))
duration = kwargs.get('duration', 60)
if 'duration' in kwargs:
cst.start_traffic(**kwargs)
else:
cst.start_traffic(duration=duration, **kwargs)
time.sleep(int(duration))
maximum_percent = kwargs.get('maximum_rx_percentage', 100.5)
minimum_percent = kwargs.get('minimum_rx_percentage', 99)
t.log("expecting received traffic will be {} - {} percent in LI".
format(minimum_percent, maximum_percent))
cst.verify_traffic(**kwargs)
for subscriber in [obj1, obj2]:
if subscriber.role == 'add' and result[0]:
subscriber.role = 'delete'
subscriber.active = True
subscriber.criteria = result[0]
else:
subscriber.role = 'add'
subscriber.active = False
subscriber.criteria = []
t.log(
"after iteration #{}, current group1 role is {}, group2 role is {}"
.format(iteration, obj1.role, obj2.role))
def mxvc_gres_test(**kwargs):
"""
GRES test for mxvc
:param kwargs:
device_id: device name, e.g.'r0'
new_mesh: create new traffic mesh(True/False)
subs: subs involved in traffic (see add_susbcriber_mesh)
duration: traffic running time, by default is 60 seconds
traffic_args: dictionary for creating traffic which include name, rate, frame_size, etc
(check add_subscriber_mesh in cstutils.py)
mininum_rx_percentage: minimum percentage for received traffic
maximum_rx_percentage: maximum percentage for received traffic
gres_type: gres type, global/local/localbackup
gres_method: gres method, (cli/reboot for global, cli/kernel_crash/scb_failover for local,
default is cli)
global_gres_method: 'cli', 'kernel_crash', 'scb_failover', default is cli
post_action_wait_time: wait time in seconds, default is 600
sleep_between_iteration: wait time in seconds between iterations, default is 600
health_check: collect health check info, default is False
iteration: iteration count, default is 1
:return:
"""
post_action_wait_time = int(kwargs.get('post_action_wait_time', 600))
sleep_between_iteration = int(kwargs.get('sleep_between_iteration', 600))
dtcp_test = kwargs.get('dtcp_test', False)
device_id = kwargs.get('device_id',
bbe.get_devices(device_tags='dut', id_only=True)[0])
router = t.get_handle(device_id)
iteration = kwargs.get('iteration', 1)
gres_type = kwargs.get('gres_type', 'global')
gres_method = kwargs.get('gres_method', 'cli')
timeout = kwargs.get('gres_check_timeout', 1800)
initial_master_re = cst.get_master_re_name(device_id)
router.cli(command='show virtual-chassis status')
cst.prepare_subscriber_traffic(**kwargs)
cst.start_traffic()
cst.get_router_sub_summary(device_id)
if dtcp_test:
dtcp.dtcp_delete_li_trigger()
dtcp.dtcp_add_li_trigger()
cst.check_fpc(device_id=device_id)
cst.get_vcp_ports(device_id=device_id)
count = 1
while count <= iteration:
t.log("checking GRES ready state before test in iteration {}".format(
count))
cst.check_gres_ready(device_id, check_timeout=timeout)
command_list = [
'show database-replication summary', 'show system relay group'
]
### log something before GRES
for command in command_list:
router.cli(command=command)
health_check = kwargs.get('health_check', False)
if health_check:
cst.get_re_fpc_memory()
hk.healthcheck_pfe_resource_monitor()
hk.healthcheck_get_task_memory()
hk.healthcheck_run_pfe_command(
command='show pfe manager session statistics')
command_list = [
'show chassis fpc', 'show virtual-chassis vc-port',
'show virtual-chassis status', 'show route summary',
'show route forwarding-table summary'
]
### log something before GRES
for command in command_list:
router.cli(command=command)
t.log("starting MXVC GRES test with type {} method {}".format(
gres_type, gres_method))
if gres_type == 'global':
if gres_method == 'cli':
t.log("Executing a Global GRES on the master by CLI.")
router.cli(
command=
"request virtual-chassis routing-engine master switch",
pattern='yes,no')
router.cli(command='yes')
elif gres_method == 'kernel_crash':
#trigger GRES by kernel crash, old master will enter into db> and then recovered
host = router.current_node.current_controller.name
cst.panic_re_recover(host=host)
elif gres_method == 'scb_failover':
t.log("pull out scb card")
input("PRESS ENTER TO CONTINUE.")
time.sleep(600)
t.log("push in scb card")
input("PRESS ENTER TO CONTINUE.")
if 'local' in gres_type:
if 'localbackup' in gres_type:
t.log("switch controller to the Vc-Bm before test")
master_node = initial_master_re.split(sep='-')[0]
backup_node = 'member1' if 'primary' in initial_master_re else 'primary'
for rename in ['re0', 're1']:
router.set_current_controller(system_node=backup_node,
controller=rename)
if router.current_node.current_controller.is_msater():
break
if gres_method == 'reboot':
t.log("perform GRES type {} using {}".format(
gres_type, gres_method))
router.cli(command="request system reboot local",
pattern='yes,no')
router.cli(command='yes')
if gres_method == 'cli':
t.log("perform GRES type {} using {}".format(
gres_type, gres_method))
resp = router.cli(
command="request chassis routing-engine master switch",
pattern='yes,no').resp
if 'error' in resp or 'not honored' in resp:
raise Exception("failed to execute local switch")
router.cli(command='yes')
time.sleep(post_action_wait_time)
router.reconnect(all, timeout=600)
master_node = router.detect_master_node()
new_master_re = cst.get_master_re_name(device_id)
t.log("current master re after GRES test is {}")
if new_master_re == initial_master_re:
if 'localbackup' not in gres_type:
t.log(
'ERROR',
'GRES failed since master RE did not switch, new master is {}'
.format(new_master_re))
else:
t.log("localbackup switched successfully")
else:
t.log("GRES switched successfully, new master is {}".format(
new_master_re))
master_re = new_master_re.split(sep='-')[1]
router.set_current_controller(system_node=master_node,
controller=master_re)
BBEJunosUtil.cpu_settle(cpu_threshold=20,
idle_min=int(kwargs.get('cpu_settle', '75')),
dead_time=1200,
interval=20)
cst.check_fpc(device_id=device_id)
cst.get_vcp_ports(device_id=device_id)
cst.get_router_sub_summary(device_id)
cst.check_link_status(router=device_id)
cst.stop_traffic()
cst.verify_traffic(**kwargs)
if dtcp_test:
dtcp.dtcp_list_li_trigger(**kwargs)
count += 1
time.sleep(sleep_between_iteration)
if health_check:
cst.get_re_fpc_memory()
hk.healthcheck_pfe_resource_monitor()
hk.healthcheck_get_task_memory()
hk.healthcheck_run_pfe_command(
command='show pfe manager session statistics')
if dtcp_test:
dtcp.dtcp_delete_li_trigger()
def dtcp_logout_login_with_same_trigger_test(**kwargs):
"""
This testcase can be used only when X-UserName is used to add triggers, and all subscribers with unique user name
:param kwargs:
device_id: router device id e.g."r0"
max_trigger_count: maximum trigger count, by default is 100, router support maximum 1024
no_trigger_change: True or False, default is True(add trigger once)
minimum_rx_percentage: minimum percentage for received traffic
maximum_rx_percentage: maximum percentage for received traffic
:return:
"""
t.log("start dtcp logout login test with same trigger")
max_count = int(kwargs.get('max_trigger_count', 100))
t.log("trying to remove existing LI triggers")
result = dtcp_list_li_trigger(**kwargs, noraise=True)
if isinstance(result, list):
dtcp_delete_li_trigger(criteria_id=result, **kwargs)
t.log("find out the current li candidates")
response = get_dtcp_li_candidates(**kwargs)
if len(response['username']) < max_count:
max_count = len(response['username'])
users = response['username']
picked_triggers = random.sample(users, max_count)
kwargs['trigger_list'] = users
tester = t.get_handle(kwargs.get('rtdevice_id', 'rt0'))
t.log("the picked triggers are {}".format(picked_triggers))
no_trigger_change = kwargs.get('no_trigger_change', True)
for iteration in range(1, int(kwargs.get('iteration', 1)) + 1):
if iteration == 1 and no_trigger_change:
t.log("adding dtcp triggers for one time only")
dtcp_add_li_trigger(trigger_type='username', **kwargs)
if not no_trigger_change:
t.log("deleting dtcp triggers in iteration {}".format(iteration))
dtcp_delete_li_trigger(**kwargs)
t.log("logout client in iteration {}".format(iteration))
cst.cst_release_clients(**kwargs)
if not no_trigger_change:
t.log("adding dtcp triggers in iteration {}".format(iteration))
dtcp_add_li_trigger(trigger_type='username', **kwargs)
time.sleep(10)
t.log("starting to login clients in iteration {}".format(iteration))
cst.cst_start_clients(**kwargs)
new_mesh = kwargs.get('new_mesh', True)
duration = kwargs.get('duration', 60)
if new_mesh:
t.log('remove existing traffic mesh')
tester.invoke('traffic_action', action='delete')
time.sleep(5)
t.log(
'add new traffic mesh for subs in iteration {}'.format(iteration))
cst.add_subscriber_mesh(**kwargs)
if 'duration' in kwargs:
cst.start_traffic(**kwargs)
else:
cst.start_traffic(duration=duration, **kwargs)
time.sleep(int(duration))
t.log('verify traffic in iteration {}'.format(iteration))
maximum_percent = kwargs.get('maximum_rx_percentage', 100.5)
minimum_percent = kwargs.get('minimum_rx_percentage', 99)
t.log(
"expecting received traffic will be {} - {} percent in LI".format(
minimum_percent, maximum_percent))
kwargs['mode'] = 'summary'
cst.verify_traffic(**kwargs)
BBEJunosUtil.cpu_settle(cpu_threshold=50,
idle_min=int(kwargs.get('cpu_settle', 60)),
dead_time=1200,
interval=20)
def mxvc_blackout_period_test(**kwargs):
"""
The MXVC blackout period test, must have dhcp clients, which used to test blackout
:param kwargs:
device_id: device name, e.g.'r0'
new_mesh: create new traffic mesh(True/False), default is true
subs: subs involved in traffic (see add_susbcriber_mesh)
duration: traffic running time, by default is 60 seconds
remove_traffic_after_test: remove traffic items after test, by default is True
traffic_args: dictionary for creating traffic which include name, rate, frame_size, etc
(check add_subscriber_mesh in cstutils.py)
mininum_rx_percentage: minimum percentage for received traffic
maximum_rx_percentage: maximum percentage for received traffic
:return:
"""
device_id = kwargs.get('device_id',
bbe.get_devices(device_tags='dut', id_only=True)[0])
router = t.get_handle(device_id)
dtcp_test = kwargs.get('dtcp_test', False)
if dtcp_test:
dtcp.dtcp_delete_li_trigger()
dtcp.dtcp_add_li_trigger()
cst.prepare_subscriber_traffic(**kwargs)
cst.start_traffic()
initial_master_re = cst.get_master_re_name(device_id)
clients_info = cst.get_router_sub_summary(device_id)
t.log("subscriber info before blackout period test is {}".format(
clients_info))
cst.get_vcp_ports(device_id=device_id)
cst.check_fpc(device_id=device_id)
timeout = kwargs.get('gres_check_timeout', 1800)
t.log("releasing the dhcp susbcribers for blackout test")
subs = bbe.get_subscriber_handles(protocol='dhcp')
t.log("checking GRES ready state before test")
cst.check_gres_ready(device_id, check_timeout=timeout)
cst.cst_release_clients(subs=subs)
health_check = kwargs.get('health_check', False)
if health_check:
cst.get_re_fpc_memory()
hk.healthcheck_pfe_resource_monitor()
hk.healthcheck_get_task_memory()
hk.healthcheck_run_pfe_command(command='show heap 0 accounting pc')
hk.healthcheck_run_pfe_command(
command='show pfe manager session statistics')
command_list = [
'show chassis fpc', 'show virtual-chassis vc-port',
'show virtual-chassis status', 'show route summary',
'show route forwarding-table summary'
]
### log something before GRES
for command in command_list:
router.cli(command=command)
rt_output = cst.get_rt_subs_info()
base_up_subs_in_rt = rt_output['rt_sessions_up']
base_down_subs_in_rt = rt_output['rt_sessions_down']
result = cst.get_configured_subs(subs=subs)
expected_subs_in_rt = result['expected_total_session_in_testers']
t.log("starting GRES, and then rebinding subscribers")
router.cli(command="request virtual-chassis routing-engine master switch",
pattern='yes,no')
router.cli(command='yes')
t.log("Start dhcp clients in blackout period")
base_time = time.time()
for subscriber in subs:
subscriber.start()
retry = 0
while retry <= kwargs.get('blackout_retry', 1):
time.sleep(20)
rt_output = cst.get_rt_subs_info()
delta_up = rt_output['rt_sessions_up'] - base_up_subs_in_rt
delta_down = rt_output['rt_sessions_down'] - base_down_subs_in_rt
delta_time = time.time() - base_time
if delta_up > 0:
t.log("{} subscribers login in {} seconds after gres".format(
delta_up, delta_time))
break
else:
if (expected_subs_in_rt - delta_up -
delta_down) == 0 and delta_time < 1200:
t.log("restart clients")
for subscriber in subs:
subscriber.restart()
elif delta_time > 1200:
t.log(
'ERROR',
'Failed to login any client in blackout time {}'.format(
delta_time))
break
t.log("waiting for MXVC to finish GRES switch")
router.reconnect(all, timeout=600)
master_node = router.detect_master_node()
new_master_re = cst.get_master_re_name(device_id)
if new_master_re == initial_master_re:
t.log(
'ERROR',
'GRES failed since master RE did not switch, new master is {}'.
format(new_master_re))
master_re = new_master_re.split(sep='-')[1]
router.set_current_controller(system_node=master_node,
controller=master_re)
BBEJunosUtil.cpu_settle(cpu_threshold=20,
idle_min=int(kwargs.get('cpu_settle', '75')),
dead_time=1200,
interval=20)
cst.cst_start_clients(restart_unbound_only=True)
if health_check:
cst.get_re_fpc_memory()
hk.healthcheck_pfe_resource_monitor()
hk.healthcheck_get_task_memory()
hk.healthcheck_run_pfe_command(command='show heap 0 accounting pc')
hk.healthcheck_run_pfe_command(
command='show pfe manager session statistics')
cst.get_vcp_ports(device_id=device_id)
cst.check_fpc(device_id=device_id)
cst.get_router_sub_summary(device_id)
cst.stop_traffic()
t.log("verify traffic after blackout test")
cst.prepare_subscriber_traffic(**kwargs)
if dtcp_test:
t.log("remove dtcp trigger after test")
dtcp.dtcp_delete_li_trigger()
def mxvc_fpc_mic_reboot_test(**kwargs):
"""
The method simulates the event of fpc, mic or pic reboot in a MXVC setup
:param kwargs:
device_id: device name, e.g.'r0'
new_mesh: create new traffic mesh(True/False)
subs: subs involved in traffic (see add_susbcriber_mesh)
duration: traffic running time, by default is 60 seconds
traffic_args: dictionary for creating traffic which include name, rate, frame_size, etc
(check add_subscriber_mesh in cstutils.py)
mininum_rx_percentage: minimum percentage for received traffic
maximum_rx_percentage: maximum percentage for received traffic
method: offline/restart/panic, default is offline
component: fpc/pic/mic
slot_info: (optional), fpc slot in dictionary, e.g. {'member':'member0', 'slot': '1'}
iteration: iteration count
post_action_wait_time: wait time in seconds, default is 600
health_check: collect health check info, default is False
:return:
"""
iteration = kwargs.get('iteration', 1)
component = kwargs.get('component', 'fpc')
method = 'offline' if component != 'fpc' else kwargs.get(
'method', 'offline')
action_list = ['offline', 'online'] if method == 'offline' else [method]
post_action_wait_time = int(kwargs.get('post_action_wait_time', 600))
dtcp_test = kwargs.get('dtcp_test', False)
device_id = kwargs.get('device_id',
bbe.get_devices(device_tags='dut', id_only=True)[0])
router = t.get_handle(device_id)
if dtcp_test:
dtcp.dtcp_delete_li_trigger()
dtcp.dtcp_add_li_trigger()
health_check = kwargs.get('health_check', False)
if health_check:
cst.get_re_fpc_memory()
hk.healthcheck_pfe_resource_monitor()
hk.healthcheck_get_task_memory()
hk.healthcheck_run_pfe_command(command='show heap 0 accounting pc')
hk.healthcheck_run_pfe_command(
command='show pfe manager session statistics')
cst.prepare_subscriber_traffic(**kwargs)
cst.start_traffic()
cst.get_router_sub_summary(device_id)
pic_info = cst.get_pic_info(device_id)
chosen_list = []
while len(chosen_list) < iteration:
if 'slot_info' in kwargs:
member = kwargs['slot_info']['member']
chosen_fpc = kwargs['slot_info']['slot']
else:
member = random.choice(list(pic_info.keys()))
chosen_fpc = random.choice(list(pic_info[member].keys()))
pic_list = pic_info[member][chosen_fpc]
chosen_mic = '0'
if 'MPC' in pic_list.pop(-1):
mic_list = []
for item in pic_list:
if item in ['0', '1']:
mic_list.append('0')
if item in ['2', '3']:
mic_list.append('1')
chosen_mic = random.choice(mic_list)
chosen_pic = random.choice(pic_list)
chosen_set = (member, chosen_fpc, chosen_mic, chosen_pic)
if chosen_set not in chosen_list:
chosen_list.append(chosen_set)
t.log("the chosen list for action {} is {}".format(method, chosen_list))
count = 1
for item in chosen_list:
for action in action_list:
t.log("action {} on chosen {} in iteration #{}".format(
action, item, count))
if action == 'panic':
router.vty(command='set parser security 10',
destination='{}-fpc{}'.format(item[0], item[1]))
router.vty(command='test panic',
destination='{}-fpc{}'.format(item[0], item[1]),
pattern='(.*)')
t.log("waiting for coredump to be generated after panic test")
#time.sleep(200)
if component == 'fpc':
command = "request chassis fpc slot {} member {} {}".format(
item[1], item[0].strip('member'), action)
router.cli(command=command)
if component == 'mic':
command = "request chassis mic mic-slot {} " \
"fpc-slot {} member {} {}".format(item[2], item[1], item[0].strip('member'), action)
router.cli(command=command)
if component == 'pic':
command = "request chassis pic pic-slot {} fpc-slot {} member" \
" {} {}".format(item[3], item[1], item[0].strip('member'), action)
router.cli(command=command)
base_time = time.time()
shutdown = False
while True:
t.log("waiting for FPC/PIC state change")
time.sleep(20)
resp = router.cli(
command="show chassis fpc pic-status {} member {}".format(
item[1], item[0].strip('member'))).resp
if action in ['restart', 'panic']:
if re.search(r'Offline', resp) and not shutdown:
shutdown = True
continue
if not re.search(r'Offline', resp):
t.log("action {} finished correctly on {} fpc {}"
).format(action, item[0], item[1])
break
elif action == 'offline':
if re.search(r'Offline', resp):
t.log(
"action {} started correctly on {} fpc {}".format(
action, item[0], item[1]))
break
elif action == 'online':
if not re.search(r'Offline', resp):
t.log(
"action {} started correctly on {} fpc {}".format(
action, item[0], item[1]))
break
if time.time() - base_time > post_action_wait_time:
raise Exception(
'{} FPC {} failed to transit to the expected state'.
format(item[0], item[1]))
cst.prepare_subscriber_traffic(**kwargs)
if dtcp_test:
dtcp.dtcp_list_li_trigger(**kwargs)
if health_check:
cst.get_re_fpc_memory()
hk.healthcheck_pfe_resource_monitor()
hk.healthcheck_get_task_memory()
hk.healthcheck_run_pfe_command(command='show heap 0 accounting pc')
hk.healthcheck_run_pfe_command(
command='show pfe manager session statistics')
count += 1
if dtcp_test:
dtcp.dtcp_delete_li_trigger()
def mxvc_vcp_link_down_test(**kwargs):
"""
The method simulates the event of vcp link down
Caution, this method did not simulate the reality problem, since it use delete the port from the vcp .
the better idea is to use pswitch to build the vcp links
:param kwargs:
device_id: device name, e.g.'r0'
new_mesh: create new traffic mesh(True/False)
subs: subs involved in traffic (see add_susbcriber_mesh)
duration: traffic running time, by default is 60 seconds
remove_traffic_after_test: remove traffic items after test, by default is True
traffic_args: dictionary for creating traffic which include name, rate, frame_size, etc
(check add_subscriber_mesh in cstutils.py)
mininum_rx_percentage: minimum percentage for received traffic
maximum_rx_percentage: maximum percentage for received traffic
iteration: iteration count
style: 'single' => one down only, 'all' => all down, default is single
:return:
"""
device_id = kwargs.get('device_id',
bbe.get_devices(device_tags='dut', id_only=True)[0])
router = t.get_handle(device_id)
cst.prepare_subscriber_traffic(**kwargs)
cst.start_traffic(**kwargs)
t.log("collect vcp ports")
status = True
router.cli(command="show virtual-chassis vc-port")
master_re = cst.get_master_re_name(device_id)
master_node = master_re.split(sep='-')[0]
rename = master_re.split(sep='-')[1]
router.set_current_controller(system_node=master_node, controller=rename)
member = master_node
if master_node == 'primary':
member = 'member0'
port_list = cst.get_vcp_ports(device_id=device_id)[member]
style = kwargs.get('style', 'single')
chosen_links = port_list
if style == 'single':
chosen_links = [random.choice(port_list)]
### save vcp link state before any action
resp = []
for link in chosen_links:
resp.append(router.cli(command="show interface {} terse".format(link)))
### show subscriber info before event, on master re
router.cli(command='show subscribers summary port')
sub_sum_pre_action = cst.get_router_sub_summary_by_port(device_id)
iteration = int(kwargs.get('iteration', 1))
count = 1
status_action = {'link_down': 'delete', 'link_up': 'set'}
while count <= iteration:
for event in ['link_down', 'link_up']:
resp2 = []
for link in chosen_links:
match = re.match(r'vcp-(\d+)/(\d+)/(\d+)', link)
t.log("start action {} on link {} in iteration #{}".format(
event, link, count))
router.cli(
command=
"request virtual-chassis vc-port {} fpc-slot {} pic-slot {} port {}"
"".format(status_action[event], match.group(1),
match.group(2), match.group(3)))
time.sleep(60)
router.cli(command="show virtual-chassis vc-port")
### show subscriber info after event, on master re
router.cli(command='show subscribers summary port')
sub_sum_after_action = cst.get_router_sub_summary_by_port(
device_id)
if sub_sum_after_action != sub_sum_pre_action:
t.log(
'ERROR', "subscriber changed from {} to {}".format(
sub_sum_pre_action, sub_sum_after_action))
status = False
sub_sum_pre_action = sub_sum_after_action
cst.stop_traffic(**kwargs)
cst.verify_traffic(**kwargs)
cst.start_traffic(**kwargs)
if event == 'link_up':
resp2.append(
router.cli(
command="show interface {} terse".format(link)))
count += 1
if resp != resp2:
status = False
t.log(
'ERROR, "vcp interface state changed from {} to {} after actions'
.format(resp, resp2))
break
if not status:
cst.stop_traffic()
raise Exception(
"mxvc vcp link down test failed in iteration {}".format(count))
cst.stop_traffic()
def mxvc_chassis_reboot(**kwargs):
"""
The method simulates the event of backup/master chassis reboot in a MXVC setup
which will be used by VCMasterReboot/vcStandbyREReboot/VCMasterPowerCycle/VCBackupReboot/VCBackupPowerCycle
:param kwargs:
device_id: device name, e.g.'r0'
new_mesh: create new traffic mesh(True/False)
subs: subs involved in traffic (see add_susbcriber_mesh)
duration: traffic running time, by default is 60 seconds
traffic_args: dictionary for creating traffic which include name, rate, frame_size, etc
(check add_subscriber_mesh in cstutils.py)
mininum_rx_percentage: minimum percentage for received traffic
maximum_rx_percentage: maximum percentage for received traffic
chassis: reboot chassis, e.g. 'VC-M' or 'VC-B' or 'VC-STDBY-RE-ALL'
method: 'cli' or 'powercycle', default is cli
post_action_wait_time: wait time in seconds, default is 600
health_check: collect health check info, default is False
:return:
"""
post_action_wait_time = int(kwargs.get('post_action_wait_time', 600))
dtcp_test = kwargs.get('dtcp_test', False)
device_id = kwargs.get('device_id',
bbe.get_devices(device_tags='dut', id_only=True)[0])
router = t.get_handle(device_id)
reboot_chassis = kwargs.get('chassis', 'VC-B').upper()
method = kwargs.get('method', 'cli')
health_check = kwargs.get('health_check', False)
cst.prepare_subscriber_traffic(**kwargs)
if dtcp_test:
dtcp.dtcp_delete_li_trigger()
dtcp.dtcp_add_li_trigger()
cst.start_traffic(**kwargs)
router.cli(command='show virtual-chassis vc-port')
if health_check:
cst.get_re_fpc_memory()
hk.healthcheck_pfe_resource_monitor()
hk.healthcheck_get_task_memory()
hk.healthcheck_run_pfe_command(command='show heap 0 accounting pc')
hk.healthcheck_run_pfe_command(
command='show pfe manager session statistics')
master_node = router.detect_master_node()
cst.check_gres_ready(device_id)
###reboot chassis REs
if method == 'cli':
if reboot_chassis == 'VC-STDBY-RE-ALL':
router.cli(
command="request system reboot all-members other-routing-engine"
)
elif reboot_chassis == 'VC-M':
if master_node == 'primary':
slot = '0'
else:
slot = '1'
command = "request system reboot member {} both-routing-engines".format(
slot)
router.cli(command=command, pattern='yes,no')
router.cli(command='yes')
elif reboot_chassis == 'VC-B':
if master_node == 'primary':
slot = '1'
else:
slot = '0'
command = "request system reboot member {} both-routing-engines".format(
slot)
router.cli(command=command, pattern='yes,no')
router.cli(command='yes')
if method == 'powercycle':
hostname = router.current_node.current_controller.name
cst.power_manager(chassis=hostname, action='cycle')
t.log("waiting for {} seconds after reboot".format(post_action_wait_time))
time.sleep(post_action_wait_time)
router.reconnect(all)
master_node = router.detect_master_node()
master_re = cst.get_master_re_name(device_id).split(sep='-')[1]
router.set_current_controller(system_node=master_node,
controller=master_re)
BBEJunosUtil.cpu_settle(cpu_threshold=20,
idle_min=int(kwargs.get('cpu_settle', '75')),
dead_time=1200,
interval=20)
router.cli(command='show virtual-chassis status')
if health_check:
cst.get_re_fpc_memory()
hk.healthcheck_pfe_resource_monitor()
hk.healthcheck_get_task_memory()
hk.healthcheck_run_pfe_command(command='show heap 0 accounting pc')
hk.healthcheck_run_pfe_command(
command='show pfe manager session statistics')
t.log('verify client count after action {}'.format(method))
cst.verify_client_count(device_id=device_id)
t.log('verify client traffic after action {}'.format(method))
cst.stop_traffic(**kwargs)
cst.verify_traffic(**kwargs)
if dtcp_test:
t.log("remove dtcp trigger after action {}".format(method))
dtcp.dtcp_delete_li_trigger()
def lns_cluster_failover_test(**kwargs):
"""
Bring down one LNS in LNS cluster and verify that affected subscribers can be bound on working LNS
:param kwargs:
device_id: device name, e.g.'r0'
new_mesh: create new traffic mesh(True/False)
subs: subs involved in traffic (see add_susbcriber_mesh)
duration: traffic running time, by default is 60 seconds
traffic_args: dictionary for creating traffic which include name, rate, frame_size, etc
(check add_subscriber_mesh in cstutils.py)
mininum_rx_percentage: minimum percentage for received traffic
maximum_rx_percentage: maximum percentage for received traffic
post_action_wait_time: wait time in seconds, default is 600
sleep_between_iteration: wait time in seconds between iterations, default is 600
dtcp_test: True/False
iteration: iteration count, default is 1
:return:
"""
dut_list = kwargs.get('device_id', bbe.get_devices(device_tags='dut', id_only=True))
dtcp_test = kwargs.get('dtcp_test', False)
lac_device = bbe.get_devices(device_tags='lac', id_only=True)
lns_device = bbe.get_devices(device_tags='lns', id_only=True)
device_list = lac_device + lns_device
for iteration in range(1, int(kwargs.get('iteration', 1)) + 1):
cst.prepare_subscriber_traffic(**kwargs)
cst.start_traffic()
for device in device_list:
dev1 = t.get_handle(device)
dev1.cli(command="show service l2tp summary")
cst.get_aaa_accounting_stats(device)
lns_id = random.choice(lns_device)
router = t.get_handle(lns_id)
t.log("bring down interfaces on {} and reboot".format(lns_id))
command_list = []
for intf in bbe.get_interfaces(device=lns_id):
command_list.append('set interfaces {} disable'.format(intf.pic))
command_list.append('commit')
router.config(command_list=command_list)
router.reboot(all=True)
t.log("waiting for cpu settle in LNS")
base_time = time.time()
for device in device_list:
dev1 = t.get_handle(device)
BBEJunosUtil.set_bbe_junos_util_device_handle(dev1)
BBEJunosUtil.cpu_settle(cpu_threshold=10, idle_min=int(kwargs.get('cpu_settle', '85')),
dead_time=1200, interval=20)
initial_client = 0
for dutid in dut_list:
summary = cst.get_router_sub_summary(dutid)
initial_client += summary['client']
while True:
current_client = 0
for dutid in dut_list:
summary = cst.get_router_sub_summary(dutid)
current_client += summary['client']
if current_client != initial_client:
initial_client = current_client
else:
t.log("client count {} is in stable state".format(current_client))
break
if time.time() - base_time > 1200:
cst.stop_traffic()
raise Exception("client count is not stable after 1200s")
t.log("waiting for 60s to check the subscriber count again")
time.sleep(60)
cst.stop_traffic()
t.log("relogin unbounded clients and verify traffic")
cst.cst_start_clients(restart_unbound_only=True)
if dtcp_test:
dtcp.dtcp_delete_li_trigger()
dtcp.dtcp_add_li_trigger()
suites.unicast_traffic_test(**kwargs)
if dtcp_test:
dtcp.dtcp_list_li_trigger(**kwargs)
if dtcp_test:
dtcp.dtcp_delete_li_trigger()
def l2tp_disconnect_test(**kwargs):
"""
Disconnect all L2TP sessions/tunnel on given box either by:
clear services l2tp tunnel all
clear services l2tp session all
in yaml file, the lac/lns needs to be tagged with lac/lns, l2tp clients needs with tag l2tpx
:param kwargs:
device_id: device name, e.g.'r0'
new_mesh: create new traffic mesh(True/False)
subs: subs involved in traffic (see add_susbcriber_mesh)
duration: traffic running time, by default is 60 seconds
traffic_args: dictionary for creating traffic which include name, rate, frame_size, etc
(check add_subscriber_mesh in cstutils.py)
mininum_rx_percentage: minimum percentage for received traffic
maximum_rx_percentage: maximum percentage for received traffic
clear_by: session/tunnel, default is session
clear_on: lac/lns, default is lns
post_action_wait_time: wait time in seconds, default is 600
sleep_between_iteration: wait time in seconds between iterations, default is 600
dtcp_test: True/False
check_interval: interval to check if the subs on lac/lns are synced, default is 120s
check_tunnel_close: True/False, default is False
iteration: iteration count, default is 1
:return:
"""
interval = int(kwargs.get('check_interval', 120))
check_tunnel_close = kwargs.get('check_tunnel_close', False)
clear_by = kwargs.get('clear_by', 'session')
clear_on = kwargs.get('clear_on', 'lns')
iteration = int(kwargs.get('iteration', 1))
dut_list = kwargs.get('device_id', bbe.get_devices(device_tags='dut', id_only=True))
dtcp_test = kwargs.get('dtcp_test', False)
lac_device = bbe.get_devices(device_tags='lac', id_only=True)
lns_device = bbe.get_devices(device_tags='lns', id_only=True)
device_list = lac_device + lns_device
count = 1
while count <= iteration:
cst.prepare_subscriber_traffic(**kwargs)
cst.start_traffic()
for device in device_list:
dev1 = t.get_handle(device)
dev1.cli(command="show service l2tp summary")
cst.get_aaa_accounting_stats(device)
t.log("clearing {} on {} in iteration {}".format(clear_by, clear_on, count))
clear_on_device = bbe.get_devices(device_tags=clear_on, id_only=True)
for device in clear_on_device:
dev2 = t.get_handle(device)
dev2.cli(command="clear services l2tp {} all | no-more".format(clear_by))
time.sleep(90)
for device in device_list:
dev1 = t.get_handle(device)
BBEJunosUtil.set_bbe_junos_util_device_handle(dev1)
BBEJunosUtil.cpu_settle(cpu_threshold=10, idle_min=int(kwargs.get('cpu_settle', '85')),
dead_time=1200, interval=20)
t.log("waiting for subscriber stable after teardown l2tp subs")
total_subs = bbe.get_subscriber_handles()
l2tp_subs = bbe.get_subscriber_handles(tag='l2tp')
for item in l2tp_subs:
total_subs.remove(item)
other_subs = total_subs
result1 = cst.get_configured_subs(subs=other_subs)
basetime = time.time()
while True:
try:
cst.verify_client_count(subs=other_subs)
except:
t.log("waiting {}s to check the client counts".format(interval))
time.sleep(interval)
if time.time() - basetime > 1800:
raise Exception("clients failed to reach the specified count after 1800s")
else:
break
result = cst.get_rt_subs_info()
if result['rt_sessions_up'] != result1['expected_total_session_in_testers']:
raise Exception("subscribers count {} in tester is not the same as expected"
" {}".format(result['rt_sessions_up'], result1['expected_total_session_in_testers']))
status = True
for device in dut_list:
dev1 = t.get_handle(device)
resp = dev1.pyez('get_l2tp_summary_information').resp
tunnel_count = int(resp.findtext('l2tp-summary-table/l2tp-tunnels'))
session_count = int(resp.findtext('l2tp-summary-table/l2tp-sessions'))
if clear_by == 'tunnel' and tunnel_count > 0:
status = False
t.log('WARN', 'failed to clear tunnel in iteration {} clear by {}'.format(count, clear_by))
if clear_by == 'session' and check_tunnel_close and tunnel_count > 0:
status = False
t.log('WARN', 'failed to clear tunnel in iteration {} clear by {}'.format(count, clear_by))
if session_count > 0:
status = False
t.log('WARN', 'failed to clear session in iteration {} clear by {}'.format(count, clear_by))
cst.get_aaa_accounting_stats(device)
if not status:
raise Exception("failed to clear session or tunnel in iteration {}".format(count))
cst.stop_traffic()
if dtcp_test:
dtcp.dtcp_delete_li_trigger()
dtcp.dtcp_add_li_trigger()
cst.cst_start_clients(subs=l2tp_subs, restart_unbound_only=True)
suites.unicast_traffic_test(**kwargs)
t.log("l2tp disconnect test finished in iteration #{}".format(count))
count += 1
def daemon_actions(**kwargs):
"""
daemon_actions(daemon_list=['jdhcpd','bbe-smgd'], re_in_action='re1',action='kill')
:param kwargs:
device_id: device name e.g. 'r0'
daemon_list: a list of daemons
action: restart/coredump/kill, by default is restart
re_in_action: re name, e.g. 're0'
cpu_settle: cpu idle value, by default is 50
cpu_check: True/False, default is True
verify_traffic: True/False, by default is False
traffic_args:
:return:
"""
t.log("inside daemon_actions")
cst._check_bbe()
verify_traffic_enable = kwargs.get('verify_traffic', False)
cpu_check = kwargs.get('cpu_check', True)
device_id = kwargs.get('device_id',
bbe.get_devices(device_tags='dut', id_only=True)[0])
rt_device_id = kwargs.get('rt_device_id', 'rt0')
tester = t.get_handle(rt_device_id)
subs = kwargs.get('subs', bbe.get_subscriber_handles())
if verify_utils.convert_str_to_num_or_bool(verify_traffic_enable):
t.log('start login subs {} before test'.format(subs))
try:
cst.cst_start_clients(**kwargs)
except:
t.log("failed to bring up subscriber")
if verify_utils.convert_str_to_num_or_bool(verify_traffic_enable):
new_mesh = kwargs.get('new_mesh', True)
duration = kwargs.get('duration', 60)
if new_mesh:
t.log('remove existing traffic mesh')
tester.invoke('traffic_action', action='delete')
time.sleep(5)
t.log('add new traffic mesh for subs')
cst.add_subscriber_mesh(**kwargs)
if 'duration' in kwargs:
cst.start_traffic(**kwargs)
else:
cst.start_traffic(duration=duration, **kwargs)
time.sleep(int(duration))
cst.verify_traffic(**kwargs)
cst.start_traffic()
daemon_action_cmds = {}
daemon_action_cmds['authd'] = {
'restart_cmd': 'restart general-authentication-service',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['jdhcpd'] = {
'restart_cmd': 'restart dhcp-service',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['dcd'] = {
'restart_cmd': 'restart interface-control',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['dfwd'] = {
'restart_cmd': 'restart firewall',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['cosd'] = {
'restart_cmd': 'restart class-of-service',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['pppd'] = {
'restart_cmd': 'restart ppp',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['jpppd'] = {
'restart_cmd':
'restart ppp-service',
'cmds_before_action': [
"show route | grep Access-internal | count",
"show route terse | grep pp0 | count",
"show route | grep demux | count"
],
'cmds_after_action': [
"show route | grep Access-internal | count",
"show route terse | grep pp0 | count",
"show route | grep demux | count"
]
}
daemon_action_cmds['pppoed'] = {
'restart_cmd': 'restart pppoe',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['l2-learning'] = {
'restart_cmd': 'restart l2-learning',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['l2cpd-service'] = {
'restart_cmd': 'restart l2cpd-service',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['jl2tpd'] = {
'restart_cmd': 'restart l2tp-universal-edge',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['dfcd'] = {
'restart_cmd': 'restart dynamic-flow-capture',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['autoconfd'] = {
'restart_cmd': 'restart auto-configuration',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['bbe-smgd'] = {
'restart_cmd': 'restart smg-service',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['pfed'] = {
'restart_cmd': 'restart statistics-service',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['smid'] = {
'restart_cmd': 'restart subscriber-management',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['bdbrepd'] = {
'restart_cmd': 'restart database-replication',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['mib2d'] = {
'restart_cmd': 'restart mib-process',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['snmpd'] = {
'restart_cmd': 'restart snmp',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['eventd'] = {
'restart_cmd': 'restart event-processing',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['essmd'] = {
'restart_cmd': 'restart extensible-subscriber-services',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['rpd'] = {
'restart_cmd': 'restart routing immediately',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['diameterd'] = {
'restart_cmd': 'restart diameter-service immediately',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['lacpd'] = {
'restart_cmd': 'restart lacp',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['pppoed'] = {
'restart_cmd': 'restart pppoe',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['chassisd'] = {
'restart_cmd': 'restart chassis-control',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['ancpd'] = {
'restart_cmd': 'restart ancpd-service',
'cmds_before_action': [],
'cmds_after_action': []
}
daemon_action_cmds['bbe-statsd'] = {
'restart_cmd': 'restart bbe-stats-service',
'cmds_before_action': [],
'cmds_after_action': []
}
if 'nfx' in t.get_system(device_id)['primary']['model']:
daemon_action_cmds['chassisd'][
'restart_cmd'] = 'restart chassis-control soft'
if 'mx204' in t.get_system(device_id)['primary']['model']:
daemon_action_cmds['chassisd'][
'restart_cmd'] = 'restart chassis-control soft'
default_list = list(daemon_action_cmds.keys())
daemon_list = kwargs.get('daemon_list', default_list)
if not isinstance(daemon_list, list):
daemon_list = [daemon_list]
device = kwargs.get('device_id',
bbe.get_devices(device_tags='dut', id_only=True)[0])
router = t.get_handle(device)
default_cmds_before_action = [
'show subscribers summary', 'show subscribers summary port'
]
default_cmds_after_action = default_cmds_before_action
master_re_name = cst.get_master_re_name(device)
re_name = kwargs.get('re_in_action', master_re_name)
if re_name != master_re_name:
if hasattr(router, 'vc') and router.vc:
master_node = re_name.split(sep='-')[0]
rename = re_name.split(sep='-')[1]
router.set_current_controller(controller=rename,
system_node=master_node)
else:
router.set_current_controller(controller=re_name,
system_node='primary')
t.log('will start action in re {}'.format(re_name))
action = kwargs.get('action', 'restart')
for daemon in daemon_list:
if daemon not in daemon_action_cmds:
t.log(
"WARN",
"daemon {} is not defined in daemon_action_cmds, will be skipped"
.format(daemon))
continue
resp = router.shell(command='ps -aux |grep {}'.format(daemon)).resp
if not re.search('/' + daemon, resp):
t.log("daemon {} is not running, will be skipped from the test".
format(daemon))
continue
cmds_before_action = daemon_action_cmds[daemon][
'cmds_before_action'] or default_cmds_before_action
for cmds in cmds_before_action:
router.cli(command=cmds, timeout=300)
time.sleep(5)
if action == 'restart':
t.log("daemon {} will be restarted".format(daemon))
router.cli(command=daemon_action_cmds[daemon]['restart_cmd'])
if action in ['coredump', 'kill']:
command = "ps axf | grep {} | grep -v grep".format(daemon)
resp = router.shell(command=command).resp
if re.match(r'\s?(\d+)\s+', resp):
pid = re.match(r'\s?(\d+)\s+', resp).group(1)
else:
t.log("daemon {} is not in the system, will ignore it".format(
daemon))
continue
router.su()
if action == 'coredump':
# Delete any daemon core if exists
router.cli(
command='file delete /var/tmp/{}.core*'.format(daemon))
# Kill daemon with SIGABRT
router.shell(command="kill -6 " + pid)
# Check daemon is restarted with new pid
restarted_with_new_pid = False
new_pid_retries = 25
new_pid_interval = 5
new_pid = pid
while new_pid_retries > 0:
time.sleep(new_pid_interval)
resp = router.shell(command=command).resp
m = re.match(r'\s?(\d+)\s+', resp)
if m is not None:
new_pid = m.group(1)
else:
t.log("daemon {} is not started yet".format(daemon))
if m is not None and new_pid != pid:
restarted_with_new_pid = True
t.log("daemon {} restarted with new pid {}".format(
daemon, new_pid))
break
new_pid_retries -= 1
if restarted_with_new_pid is False:
t.log("{} is not restarted after {} seconds".format(
daemon, new_pid_retries * new_pid_interval))
raise Exception("daemon was not restarted")
# Check daemon core dump
core_retries = 100
core_interval = 5
daemon_core_found = False
while core_retries > 0:
resp = router.shell(
command="ls /var/tmp/{}.core*".format(daemon)).resp
if not re.search(daemon, resp):
core_retries -= 1
else:
daemon_core_found = True
t.log("core dump was generated for daemon {}".format(
daemon))
break
time.sleep(core_interval)
if daemon_core_found is False:
raise Exception(
"no core was generated for daemon {}".format(daemon))
# Delete generated daemon core file
# ywang tested on 19.2 that when daemon is core dumping, the helper is dumpd.
dump_completed = False
dump_retries = 100
dump_interval = 10
dump_cmd = "ps axf | grep dumpd | grep -v grep"
while dump_retries > 0:
resp = router.shell(command=dump_cmd).resp
if re.search('dumpd', resp, re.MULTILINE):
time.sleep(dump_interval)
dump_retries -= 1
continue
else:
dump_completed = True
break
if dump_completed:
router.cli(
command='file delete /var/tmp/{}.core*'.format(daemon))
t.log("Deleted core dump generated by {}".format(daemon))
else:
t.log("{} core dump is not completed after {} seconds".
format(daemon, dump_retries * dump_interval))
raise Exception("daemon core dump takes too long")
if action == 'kill':
router.shell(command="kill -9 " + pid)
time.sleep(5)
resp = router.shell(command=command).resp
new_pid = re.match(r'\s?(\d+)\s+', resp).group(1)
if new_pid == pid:
#router.set_current_controller(controller=master_re_name, system_node='primary')
raise Exception("daemon {} was not killed".format(daemon))
else:
t.log("daemon {} was killed, new pid is {}".format(
daemon, new_pid))
if cpu_check:
BBEJunosUtil.cpu_settle(
cpu_threshold=int(kwargs.get('cpu_process', 30)),
idle_min=int(kwargs.get('cpu_settle', '75')),
dead_time=int(kwargs.get('cpu_deadtime', 1200)),
interval=int(kwargs.get('cpu_interval', 20)))
check_client = kwargs.get('verify_client', False)
if verify_utils.convert_str_to_num_or_bool(check_client):
cst.verify_client_count(**kwargs)
cmds_after_action = default_cmds_after_action if not daemon_action_cmds[daemon]['cmds_after_action'] else \
daemon_action_cmds[daemon]['cmds_after_action']
for cmds in cmds_after_action:
router.cli(command=cmds, timeout=300)
if verify_utils.convert_str_to_num_or_bool(verify_traffic_enable):
cst.stop_traffic()
cst.verify_traffic(**kwargs)
if daemon != daemon_list[-1]:
cst.start_traffic()
if re_name != master_re_name:
router.set_current_controller(controller=master_re_name,
system_node='primary')
t.log('switch back to master re {}'.format(master_re_name))
