def qos_fast_reboot_module_hooks(request): # add things at the start of this module global vars vars = st.ensure_min_topology("D1T1:1") st.log("Configuring supported QoS features") wred_data = wred_config.init_vars(vars) st.log('Creating WRED and ECN table') putils.exec_all(True, [ putils.ExecAllFunc(apply_wred_ecn_config, vars.D1, wred_data['wred_ecn_json_config']) ]) st.log("Checking for wred config before save and fast-reboot") wred_verify() st.log("checking for ecn config before save and fast-reboot") ecn_verify() st.log("Configuring IPV4 ACL with rule") ipv4_acl_config() st.log("Configuring IPV6 ACL with rule") ipv6_acl_config() st.log("Checking for IPV4 ACL config before save and fast-reboot") ipv4_acl_verify() st.log("Checking for IPV6 ACL config before save and fast-reboot") ipv6_acl_verify() st.log("Configuring COS") cos_config() st.log("Checking for COS config before save and fast-reboot") cos_config_verify() yield # add things at the end of this module" #Below step will clear COS, WRED and ECN config from the device. qos_obj.clear_qos_config(vars.D1) #Below step will clear all ACL config from the device. acl_obj.clear_acl_config(vars.D1)
def ip_unconfig(): def f1(): ip_obj.config_ip_addr_interface(data.d1, data.d1_d2_intf_1, data.d1_d2_intf_1_ip, data.def_mask_ip, 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d1, 'Vlan121', data.d1_d2_vlan_1_ip, data.def_mask_ip, 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d1, data.d1_d4_pc_1, data.d1_d4_pc_1_ip, data.def_mask_ip, 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d1, data.d1_d2_intf_1, data.d1_d2_intf_1_ip6, data.def_mask_ip6, 'ipv6',config='remove') ip_obj.config_ip_addr_interface(data.d1, 'Loopback1', '1.1.1.1', '32', 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d1, 'Loopback2', '1.1.1.2', '32', 'ipv4',config='remove') def f2(): ip_obj.config_ip_addr_interface(data.d2, data.d2_d1_intf_1, data.d2_d1_intf_1_ip, data.def_mask_ip, 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d2, 'Vlan121', data.d2_d1_vlan_1_ip, data.def_mask_ip, 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d2, data.d2_d3_pc_1, data.d2_d3_pc_1_ip, data.def_mask_ip, 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d2, data.d2_d1_intf_1, data.d2_d1_intf_1_ip6, data.def_mask_ip6, 'ipv6',config='remove') ip_obj.config_ip_addr_interface(data.d2, 'Loopback1', '2.2.2.1', '32', 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d2, 'Loopback2', '2.2.2.2', '32', 'ipv4',config='remove') def f3(): ip_obj.config_ip_addr_interface(data.d3, data.d3_d4_intf_1, data.d3_d4_intf_1_ip, data.def_mask_ip, 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d3, 'Vlan343', data.d3_d4_vlan_1_ip, data.def_mask_ip, 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d3, data.d3_d2_pc_1, data.d3_d2_pc_1_ip, data.def_mask_ip, 'ipv4',config='remove') #ip_obj.config_ip_addr_interface(data.d3, data.d3_d4_intf_1, data.d3_d4_intf_1_ip6, data.def_mask_ip6, 'ipv6',config='remove') ip_obj.config_ip_addr_interface(data.d3, 'Loopback1', '3.3.3.1', '32', 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d3, 'Loopback2', '3.3.3.2', '32', 'ipv4',config='remove') def f4(): ip_obj.config_ip_addr_interface(data.d4, data.d4_d3_intf_1, data.d4_d3_intf_1_ip, data.def_mask_ip, 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d4, 'Vlan343', data.d4_d3_vlan_1_ip, data.def_mask_ip, 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d4, data.d4_d1_pc_1, data.d4_d1_pc_1_ip, data.def_mask_ip, 'ipv4',config='remove') #ip_obj.config_ip_addr_interface(data.d4, data.d4_d3_intf_1, data.d4_d3_intf_1_ip6, data.def_mask_ip6, 'ipv6',config='remove') ip_obj.config_ip_addr_interface(data.d4, 'Loopback1', '4.4.4.1', '32', 'ipv4',config='remove') ip_obj.config_ip_addr_interface(data.d4, 'Loopback2', '4.4.4.2', '32', 'ipv4',config='remove') putils.exec_all(True, [[f1], [f2], [f3], [f4]])
def vlan_unconfig(): def f1(): vlan_obj.delete_vlan_member(data.d1,'121',data.d1_d2_intf_2) vlan_obj.delete_vlan(data.d1, '121') def f2(): vlan_obj.delete_vlan_member(data.d2,'121',data.d2_d1_intf_2) vlan_obj.delete_vlan(data.d2, '121') def f3(): vlan_obj.delete_vlan_member(data.d3,'343',data.d3_d4_intf_2) vlan_obj.delete_vlan(data.d3, '343') def f4(): vlan_obj.delete_vlan_member(data.d4,'343',data.d4_d3_intf_2) vlan_obj.delete_vlan(data.d4, '343') putils.exec_all(True, [[f1], [f2], [f3], [f4]])
def print_debug(): def f1(): pc_obj.verify_portchannel(data.d1, data.d1_d4_pc_1) ip_obj.ping(data.d1, data.d2_d1_vlan_1_ip) ip_obj.ping(data.d1, data.d4_d1_pc_1_ip) def f2(): pc_obj.verify_portchannel(data.d2, data.d2_d3_pc_1) ip_obj.ping(data.d3, data.d4_d3_intf_1_ip) ip_obj.ping(data.d3, data.d4_d3_vlan_1_ip) ip_obj.ping(data.d3, data.d3_d3_pc_1_ip) def f3(): pc_obj.verify_portchannel(data.d3, data.d3_d2_pc_1) def f4(): pc_obj.verify_portchannel(data.d4, data.d4_d1_pc_1) putils.exec_all(True, [[f1], [f2], [f3], [f4]])
def static_route_unconfig(): def f1(): ip_obj.delete_static_route(data.d1, data.d2_d1_intf_1_ip, '2.2.2.0/30', family='ipv4', shell="vtysh") ip_obj.delete_static_route(data.d1, data.d2_d1_vlan_1_ip, '2.2.2.0/30', family='ipv4', shell="vtysh") ip_obj.delete_static_route(data.d1, data.d4_d1_pc_1_ip, '4.4.4.0/30', family='ipv4', shell="vtysh") def f2(): ip_obj.delete_static_route(data.d2, data.d1_d2_intf_1_ip, '1.1.1.0/30', family='ipv4', shell="vtysh") ip_obj.delete_static_route(data.d2, data.d1_d2_vlan_1_ip, '1.1.1.0/30', family='ipv4', shell="vtysh") ip_obj.delete_static_route(data.d2, data.d3_d2_pc_1_ip, '3.3.3.0/30', family='ipv4', shell="vtysh") def f3(): ip_obj.delete_static_route(data.d3, data.d3_d4_intf_1_ip, '4.4.4.0/30', family='ipv4', shell="vtysh") ip_obj.delete_static_route(data.d3, data.d3_d4_vlan_1_ip, '4.4.4.0/30', family='ipv4', shell="vtysh") ip_obj.delete_static_route(data.d3, data.d2_d3_pc_1_ip, '2.2.2.0/30', family='ipv4', shell="vtysh") def f4(): ip_obj.delete_static_route(data.d4, data.d4_d3_intf_1_ip, '3.3.3.0/30', family='ipv4', shell="vtysh") ip_obj.delete_static_route(data.d4, data.d4_d3_vlan_1_ip, '3.3.3.0/30', family='ipv4', shell="vtysh") ip_obj.delete_static_route(data.d4, data.d1_d4_pc_1_ip, '1.1.1.0/30', family='ipv4', shell="vtysh") putils.exec_all(True, [[f1], [f2], [f3], [f4]])
def qos_warm_reboot_module_hooks(request): # add things at the start of this module init_vars() initialize_variables() get_parms() st.log("Checking whether the platform supports warm-reboot") if not data.platform.lower( ) in data.constants['WARM_REBOOT_SUPPORTED_PLATFORMS']: st.report_unsupported('test_case_unsupported') st.log("Configuring supported QoS features") wred_data = wred_config.init_vars(vars) st.log('Creating WRED and ECN table') putils.exec_all(True, [ putils.ExecAllFunc(apply_wred_ecn_config, vars.D1, wred_data['wred_ecn_json_config']) ]) st.log("Checking for wred config before save and warm-reboot") wred_verify() st.log("checking for ecn config before save and warm-reboot") ecn_verify() st.log("Configuring IPV4 ACL with rule") ipv4_acl_config() st.log("Configuring IPV6 ACL with rule") ipv6_acl_config() st.log("Checking for IPV4 ACL config before save and warm-reboot") ipv4_acl_verify() st.log("Checking for IPV6 ACL config before save and warm-reboot") ipv6_acl_verify() st.log("Configuring COS") cos_config() st.log("Checking for COS config before save and warm-reboot") cos_config_verify() yield # add things at the end of this module" #Below step will clear COS, WRED and ECN config from the device. qos_obj.clear_qos_config(vars.D1) #Below step will clear all ACL config from the device. acl_obj.clear_acl_config(vars.D1)
def evpn_cleanup(): st.log("Delete L2 vlan to VNI mapping") putils.exec_all(True, [[ map_vlan_vni, data.d2, "vtepLeaf1", "100", "100", "1", "no", False, data.cli_type ], [ map_vlan_vni, data.d3, "vtepLeaf2", "100", "100", "1", "no", False, data.cli_type ]]) st.log("Remove evpn nvo instance from all leaf nodes") putils.exec_all(True, [[ create_evpn_instance, data.d2, "nvoLeaf1", "vtepLeaf1", "no", False, data.cli_type ], [ create_evpn_instance, data.d3, "nvoLeaf2", "vtepLeaf2", "no", False, data.cli_type ]]) st.log("Remove vtep from all leaf nodes") putils.exec_all(True, [[ create_overlay_intf, data.d2, "vtepLeaf1", "3.3.3.2", "no", False, data.cli_type ], [ create_overlay_intf, data.d3, "vtepLeaf2", "4.4.4.2", "no", False, data.cli_type ]])
def portchannel_unconfig(): def f1(): pc_obj.add_del_portchannel_member(data.d1, data.d1_d4_pc_1, data.d1_d4_intf_1, 'del') pc_obj.add_del_portchannel_member(data.d1, data.d1_d4_pc_1, data.d1_d4_intf_2, 'del') pc_obj.delete_portchannel(data.d1, data.d1_d4_pc_1) def f2(): pc_obj.add_del_portchannel_member(data.d2, data.d2_d3_pc_1, data.d2_d3_intf_1, 'del') pc_obj.add_del_portchannel_member(data.d2, data.d2_d3_pc_1, data.d2_d3_intf_2, 'del') pc_obj.delete_portchannel(data.d2, data.d2_d3_pc_1) def f3(): pc_obj.add_del_portchannel_member(data.d3, data.d3_d2_pc_1, data.d3_d2_intf_1, 'del') pc_obj.add_del_portchannel_member(data.d3, data.d3_d2_pc_1, data.d3_d2_intf_2, 'del') pc_obj.delete_portchannel(data.d3, data.d3_d2_pc_1) def f4(): pc_obj.add_del_portchannel_member(data.d4, data.d4_d1_pc_1, data.d4_d1_intf_1, 'del') pc_obj.add_del_portchannel_member(data.d4, data.d4_d1_pc_1, data.d4_d1_intf_2, 'del') pc_obj.delete_portchannel(data.d4, data.d4_d1_pc_1) putils.exec_all(True, [[f1], [f2], [f3], [f4]])
def verify_ping(): def f1(): ping_res = ip_obj.ping(data.d1, data.lbk_ip_list[1][0], timeout=7) return ping_res def f2(): ping_res = ip_obj.ping(data.d2, data.lbk_ip_list[2][0], timeout=7) return ping_res def f3(): ping_res = ip_obj.ping(data.d3, data.lbk_ip_list[3][0], timeout=7) return ping_res [res, _] = putils.exec_all(True, [[f1], [f2], [f3]]) if res[0] and res[1] and res[2]: return True return False
def prologue_epilogue(): if st.get_ui_type() in ['klish']: st.report_unsupported( 'test_execution_skipped', 'Skipping Chef_MCLAG test case for ui_type={}'.format( st.get_ui_type())) st.log('Define Common config, including TGEN related, if any') initialize_topology() loc_lib.chef_pre_config(data.d1, data.d1_ip) if not chef_evpn_obj.sync_with_server_time( data.my_dut_list, chef_server.ip, chef_server.username, chef_server.password): st.report_env_fail("test_case_not_executeds") f1 = lambda x: chef_evpn_obj.delete_client_pem_files( data.d1, chef_server.client_path) f2 = lambda x: chef_evpn_obj.delete_client_pem_files( data.d2, chef_server.client_path) f3 = lambda x: chef_evpn_obj.delete_client_pem_files( data.d3, chef_server.client_path) putils.exec_all(True, [[f1, 1], [f2, 1], [f3, 1]]) chef_evpn_obj.generate_certs(chef_server.ssh_obj, chef_server.path) # Cleanup exisitng node if any chef_evpn_obj.delete_chef_node(chef_server.ssh_obj, ' '.join(data.node_list_mc), ' '.join(data.role_list_mc)) # Generate certs and bootstrap node chef_evpn_obj.generate_certs(chef_server.ssh_obj, chef_server.path) if not chef_evpn_obj.bootstrap_chef_node( chef_server.ssh_obj, chef_server.path, data.d1_ip, 'admin', 'broadcom', data.node_list_mc[0]): st.report_env_fail("chef_bootstrap_fail") if not chef_evpn_obj.bootstrap_chef_node( chef_server.ssh_obj, chef_server.path, data.d2_ip, 'admin', 'broadcom', data.node_list_mc[1]): st.report_env_fail("chef_bootstrap_fail") if not chef_evpn_obj.bootstrap_chef_node( chef_server.ssh_obj, chef_server.path, data.d3_ip, 'admin', 'broadcom', data.node_list_mc[2]): st.report_env_fail("chef_bootstrap_fail") # upload cookbook #chef_evpn_obj.upload_chef_cookbook(chef_server.ssh_obj, chef_server.path) yield st.log('Define Common cleanup, including TGEN related, if any') for role, node_name in zip(data.role_list_mc, data.node_list_mc): run_list = 'role[{}],recipe[sonic::vlan],recipe[sonic::lag],recipe[sonic::interface],recipe[sonic::mclag]'.format( role) chef_evpn_obj.update_node_run_list(chef_server.ssh_obj, node_name, run_list, 'remove') # Cleanup exisitng node if any chef_evpn_obj.delete_chef_node(chef_server.ssh_obj, ' '.join(data.node_list_mc), ' '.join(data.role_list_mc))
def run_chef_all_nodes(): f1 = lambda x: chef_evpn_obj.run_chef_client(data.d1) f2 = lambda x: chef_evpn_obj.run_chef_client(data.d2) f3 = lambda x: chef_evpn_obj.run_chef_client(data.d3) putils.exec_all(True, [[f1, 1], [f2, 1], [f3, 1]])
def test_chef_evpn_l2mclag(): result = 0 chef_server.role_dir = os.path.join(chef_server.path, 'roles') for file_name, role, node_name in zip(data.role_tc_list_l2, data.role_list_mc, data.node_list_mc): new_file_name = modify_role_jsonfile(file_name) loc_lib.copy_role_file_to_chef_server(new_file_name) st.log('{}, {}, {}'.format(file_name, role, node_name)) chef_evpn_obj.upload_role_chef_server(chef_server.ssh_obj, chef_server.role_dir, file_name=new_file_name) if file_name == "tmpl_qt_d3_tc_l2mclag.json": run_list = 'role[{}],recipe[sonic::lag],recipe[sonic::vlan],recipe[sonic::interface]'.format( role) else: run_list = 'role[{}],recipe[sonic::lag],recipe[sonic::vlan],recipe[sonic::interface],recipe[sonic::mclag]'.format( role) chef_evpn_obj.update_node_run_list(chef_server.ssh_obj, node_name, run_list) run_chef_all_nodes() st.wait(data.wait) dict1 = { 'domain_id': data.mclag_domain, 'local_ip': data.d1d2_ip, 'peer_ip': data.d2d1_ip, 'peer_link_inf': data.peerlinkintf, 'session_status': 'OK' } dict2 = { 'domain_id': data.mclag_domain, 'local_ip': data.d2d1_ip, 'peer_ip': data.d1d2_ip, 'peer_link_inf': data.peerlinkintf, 'session_status': 'OK' } [result, exceptions] = putils.exec_parallel(True, [data.d1, data.d2], mclag.verify_domain, [dict1, dict2]) if not all(i is None for i in exceptions): st.log(exceptions) if False in result: st.log('MCLAG -{} state verification FAILED'.format(data.mclag_domain)) st.report_fail('chef_mclag_state_fail') if not verify_portchannel_and_member_status(data.d3, data.portChannelName, [data.d3d1p1, data.d3d2p1], iter_count=6, iter_delay=1, state='up'): st.error("port channel {} on DUT {} state fail with {}".format( data.portChannelName, data.d1, "up")) st.report_fail('chef_mclag_state_fail') data.role_list_mc_new = [data.role_list_mc[0], data.role_list_mc[1]] data.node_list_mc_new = [data.node_list_mc[0], data.node_list_mc[1]] for role, node_name in zip(data.role_list_mc_new, data.node_list_mc_new): run_list = 'role[{}],recipe[sonic::vlan],recipe[sonic::lag],recipe[sonic::interface]'.format( role) chef_evpn_obj.update_node_run_list(chef_server.ssh_obj, node_name, run_list, 'remove') for file_name, role, node_name in zip(data.role_tc_list_mclagdel, data.role_list_mc_new, data.node_list_mc_new): loc_lib.copy_role_file_to_chef_server(file_name) chef_evpn_obj.upload_role_chef_server(chef_server.ssh_obj, chef_server.role_dir, file_name=file_name) run_list = 'role[{}],recipe[sonic::mclag]'.format(role) chef_evpn_obj.update_node_run_list(chef_server.ssh_obj, node_name, run_list) putils.exec_all(True, [[chef_evpn_obj.run_chef_client, data.d1], [chef_evpn_obj.run_chef_client, data.d2]]) st.wait(data.wait) [result, exceptions] = putils.exec_parallel(True, [data.d1, data.d2], mclag.verify_domain, [dict1, dict2]) if not all(i is None for i in exceptions): st.log(exceptions) if True in result: st.log('MCLAG -{} state verification FAILED'.format(data.mclag_domain)) st.report_fail('chef_mclag_delete_fail') st.report_pass('chef_l2mclag_pass')
def initialize_topology(): # code for ensuring min topology vars = st.ensure_min_topology('D1D2:2', 'D1D3:0', 'D1D4:2', 'D3D4:2', 'D3D2:2', 'D2D4:0') data.my_dut_list = st.get_dut_names() data.d1 = data.my_dut_list[0] data.d2 = data.my_dut_list[1] data.d3 = data.my_dut_list[2] data.d4 = data.my_dut_list[3] data.d1_d2_intf_1 = vars.D1D2P1 data.d2_d1_intf_1 = vars.D2D1P1 data.d1_d2_intf_2 = vars.D1D2P2 data.d2_d1_intf_2 = vars.D2D1P2 data.d1_d4_intf_1 = vars.D1D4P1 data.d4_d1_intf_1 = vars.D4D1P1 data.d1_d4_intf_2 = vars.D1D4P2 data.d4_d1_intf_2 = vars.D4D1P2 data.d3_d4_intf_1 = vars.D3D4P1 data.d4_d3_intf_1 = vars.D4D3P1 data.d3_d4_intf_2 = vars.D3D4P2 data.d4_d3_intf_2 = vars.D4D3P2 data.d3_d2_intf_1 = vars.D3D2P1 data.d2_d3_intf_1 = vars.D2D3P1 data.d3_d2_intf_2 = vars.D3D2P2 data.d2_d3_intf_2 = vars.D2D3P2 data.wait = 120 def f1(): ip = loc_lib.get_dut_ip(data.d1) return ip def f2(): ip = loc_lib.get_dut_ip(data.d2) return ip def f3(): ip = loc_lib.get_dut_ip(data.d3) return ip def f4(): ip = loc_lib.get_dut_ip(data.d4) return ip [res, _] = putils.exec_all(True, [[f1], [f2], [f3], [f4]]) data.d1_ip = res[0] data.d2_ip = res[1] data.d3_ip = res[2] data.d4_ip = res[3] data.role_path = os.path.join(os.path.dirname(__file__), data.role_dir) st.log('#####################{}'.format(data.role_path)) chef_param_list = [ 'ip', 'username', 'password', 'cookbook_path', 'path', 'client_path', 'validation_file', 'client_rb', 'client_log', 'user_home_folder' ] for chef_param in chef_param_list: chef_server[chef_param] = mutils.ensure_service_params( data.d1, chef_server.name, chef_param) chef_server.url = "https://{}:443".format(chef_server.ip) st.log("Chef server url used : {}".format(chef_server.url)) st.log("Logging in to chef server with the params from config file.") chef_server.ssh_obj = con_obj.connect_to_device(chef_server.ip, chef_server.username, chef_server.password) if not chef_server.ssh_obj: st.error("SSH connetion object not found.") st.report_env_fail("ssh_connection_failed", chef_server.ip) for key, value in data.items(): st.log('{} - {}'.format(key, value))
def test_chef_evpn_001(): result = 0 chef_server.role_dir = os.path.join(chef_server.path, 'roles') for file_name, role, node_name in zip(data.role_tc_list, data.role_list, data.node_list): st.log('{}, {}, {}'.format(file_name, role, node_name)) new_file_name = loc_lib.modify_role_file(file_name) #copy_role_file_to_chef_server(file_name) loc_lib.copy_role_file_to_chef_server(new_file_name) st.log('{}, {}, {}'.format(new_file_name, role, node_name)) chef_evpn_obj.upload_role_chef_server(chef_server.ssh_obj, chef_server.role_dir, file_name=new_file_name) run_list = 'role[{}],recipe[sonic::vlan],recipe[sonic::loopback],recipe[sonic::lag],recipe[sonic::vrf],recipe[sonic::interface],recipe[sonic::bgprouter]'.format( role) chef_evpn_obj.update_node_run_list(chef_server.ssh_obj, node_name, run_list) f1 = lambda x: chef_evpn_obj.run_chef_client(data.d1) f2 = lambda x: chef_evpn_obj.run_chef_client(data.d2) f3 = lambda x: chef_evpn_obj.run_chef_client(data.d3) f4 = lambda x: chef_evpn_obj.run_chef_client(data.d4) putils.exec_all(True, [[f1, 1], [f2, 1], [f3, 1], [f4, 1]]) st.wait(data.wait) if not loc_lib.retry_api( ip_bgp.check_bgp_session, data.d1, nbr_list=[data.d2_d1_vlan_1_ip, data.d4_d1_pc_1_ip], state_list=['Established'] * 2): st.error( "one or more BGP sessions did not come up between dut1 and dut2") result += 1 if not loc_lib.retry_api( ip_bgp.check_bgp_session, data.d1, nbr_list=[data.d2_d1_intf_1_ip, data.d2_d1_intf_1_ip6], state_list=['Established'] * 2, vrf_name='Vrf-01'): st.error( "one or more BGP sessions did not come up between dut1 and dut2") result += 1 if not loc_lib.retry_api(ip_bgp.check_bgp_session, data.d3, nbr_list=[ data.d4_d3_intf_1_ip, data.d4_d3_vlan_1_ip, data.d2_d3_pc_1_ip, data.d4_d3_intf_1_ip6 ], state_list=['Established'] * 4): st.error( "one or more BGP sessions did not come up between dut3 and dut4") result += 1 if not loc_lib.verify_ping(): st.error("Ping to loopback interface failed") result += 1 if result == 0: st.report_pass('test_case_passed') else: loc_lib.print_debug() st.report_fail('test_case_failed')