def test_system_time_confirmed(self): backup = ManagerProcess.SYSTEM_TIME_CONFIRM ManagerProcess._system_time_confirmed(step=t_TpcMessage.SUCCESS) result = ManagerProcess.is_system_time_confirmed() self.assertTrue(result) ManagerProcess.SYSTEM_TIME_CONFIRM = backup self.assertEquals(ManagerProcess.SYSTEM_TIME_CONFIRM, "None")
def test_handle_rcp_request_msg(self): print '*' * 80 print "start to test _handle_rcp_request_msg" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.fsm.STARTUP_DHCP_OK(interface='eth0') # msg is not string mgr._handle_rcp_request_msg('test', 1) # msg is unknown mgr._handle_rcp_request_msg('test', 'msg') # l2tp not ready ccap_core = CCAPCore.ccap_core_db.values() ccap_core[0].interface = 'eth0' ccap_core[0].ccap_core_network_address = '1.1.1.5' ccap_core[0].is_active = CoreDescription.CORE_MODE_ACTIVE ccap_core[0].is_principal = CoreDescription.CORE_ROLE_PRINCIPAL mgr._handle_rcp_request_msg('get_active_principal', 'msg') # l2tp ready ccap_core[0].agent_status[ProcessAgent.AGENTTYPE_L2TP] = True mgr._handle_rcp_request_msg('get_active_principal', 'msg') mgr.fsm.Error(msg='test fsm error')
def test_start_fake_process(self): """Start provision process with fake agent.""" print '*' * 80 print "start to test provision manager with fake agent" mgr = ManagerProcess(simulator=True) mgr.dispatcher.timer_register(10, self.end_manager, arg=mgr) mgr.start()
def setUp(self): setup_test_redis() CCAPCoreOrchestrator.NO_PRINCIPAL_CORE_FOUND_TIMEOUT = 60 CCAPCore.ccap_core_db = {} self.fake_cb_cnt = 0 self.mgr = ManagerProcess(simulator=True, test_flag=True) ManagerProcess.SYSTEM_TIME_CONFIRM = "None"
def test_seek_principal_failure_1(self): print '*' * 80 print "start to test seek_principal_failure 1" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.fsm.STARTUP_DHCP_OK(interface='eth0') mgr.principal_core_seek_failure(None) mgr.fsm.Error(msg='test fsm error')
def demoMgrProcess(): global uTMgrProcess global uTMgrApiDispatch print "demoMgrProcess thread start!" uTMgrProcess = ManagerProcess(test_flag=True) uTMgrApiDispatch = uTMgrProcess.dispatcher uTMgrProcess.start() print "demoMgrProcess thread done!"
def test_interface_scan_timeout_callback(self): print '*' * 80 print "start to test _interface_scan_timeout_callback" mgr = ManagerProcess(simulator=False) try: mgr._interface_scan_timeout_callback(None) except Exception as e: self.assertEqual(type(e), IOError) mgr.fsm.Error(msg='test fsm error')
def test_handle_mgr_8021x_event(self): print '*' * 80 print "start to test _handle_mgr_8021x_event" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.fsm.STARTUP_DHCP_OK(interface='eth0') event_request_rsp = protoDef.msg_event_notification() event_request_rsp.mgr_event.mgr_id = mgr.mgr_id event_request_rsp.mgr_event.event_id = ProcessAgent.AGENTTYPE_8021X mgr._handle_mgr_8021x_event(event_request_rsp.mgr_event) mgr.fsm.Error(msg='test fsm error')
def demoMgrProcess(): global uTMgrProcess global uTMgrApiDispatch print "demoMgrProcess thread start!" uTMgrProcess = ManagerProcess(test_flag=True) uTMgrApiDispatch = uTMgrProcess.dispatcher # Unregister interface_scan_timer to make the tearDown faster. # When unregistered we don't need to wait for the long interface scan # timeout before closing socket in tearDown. uTMgrApiDispatch.timer_unregister(uTMgrProcess.interface_scan_timer) uTMgrProcess.start() print "demoMgrProcess thread done!"
def test_interface_up_handler(self): print '*' * 80 print "start to test interface_up_handler" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.fsm.STARTUP_DHCP_OK(interface='eth0') mgr.interface_list.append({ "interface": 'eth0', "status": mgr.INTERFACE_UP, }) mgr.interface_up_handler(['eth1', 'eth2']) mgr.fsm.Error(msg='test fsm error')
def test_handle_mgr_ipsec_event(self): print '*' * 80 print "start to test _handle_mgr_ipsec_event" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.fsm.STARTUP_DHCP_OK(interface='eth0') event_request_rsp = protoDef.msg_event_notification() event_request_rsp.mgr_event.mgr_id = mgr.mgr_id event_request_rsp.mgr_event.event_id = ProcessAgent.AGENTTYPE_IPSEC mgr._handle_mgr_ipsec_event(event_request_rsp.mgr_event) keys = CCAPCore.ccap_core_db.keys() for key in keys: CCAPCore.ccap_core_db.pop(key) mgr.fsm.Error(msg='test fsm error')
def test_interface_fail(self): print '*' * 80 print "start to test interface fail" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.fsm.PROVISION_INTERFACE_FAIL(msg='test interface fail') mgr.fsm.Error(msg='test fsm error')
def test_gcp_retry_negative_case(self): print '*' * 80 print "start to test_gcp_retry_negative_case" mgr = ManagerProcess(simulator=False) core_a = CCAPCore('CORE-1000000001', CoreDescription.CORE_ROLE_PRINCIPAL, CoreDescription.CORE_MODE_ACTIVE, mgr=mgr, ccap_core_interface='eth0', ccap_core_network_address='1.1.1.2') core_b = CCAPCore('CORE-1000000002', CoreDescription.CORE_ROLE_AUXILIARY, CoreDescription.CORE_MODE_ACTIVE, mgr=mgr, ccap_core_interface='eth0', ccap_core_network_address='1.1.1.3') CCAPCore.ccap_core_db[core_a.ccap_core_id] = core_a CCAPCore.ccap_core_db[core_b.ccap_core_id] = core_b core_a.fsm.TRIGGER_Startup() core_a.fsm.TRIGGER_INTERFACE_UP() core_a.fsm.TRIGGER_MAC_8021X_OK() core_a.fsm.TRIGGER_DHCP_OK() core_a.fsm.TRIGGER_IPSEC_OK() core_a.del_ccap_core() core_b.del_ccap_core() core_a.fsm.TRIGGER_IPSEC_FAIL()
def test_seek_principal_core(self): print '*' * 80 print "start to test orchestrator seek_principal_core" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.interface_candidate.append('eth0') ccap_core = CCAPCore.ccap_core_db.values() # principal core is none mgr.core_orchestrator.seek_principal_core() # ccap_core_id not in db mgr.core_orchestrator.active_list.append(('eth0', '1.1.1.1', 'test')) mgr.core_orchestrator.seek_principal_core() # ccap_core_id in db mgr.core_orchestrator.active_list.append( ('eth0', '1.1.1.1', ccap_core[0].ccap_core_id)) mgr.core_orchestrator.seek_principal_core() # no principal active ccap_core[0].agent_status[ProcessAgent.AGENTTYPE_IPSEC] = True ccap_core[0].is_principal = CoreDescription.CORE_ROLE_NONE mgr.core_orchestrator.seek_principal_core() # principal active ccap_core[0].is_principal = CoreDescription.CORE_ROLE_PRINCIPAL mgr.core_orchestrator.seek_principal_core() second_core = CCAPCore('CORE-1234567890', CoreDescription.CORE_ROLE_PRINCIPAL, CoreDescription.CORE_MODE_ACTIVE, mgr=mgr, ccap_core_interface='eth0', ccap_core_network_address='1.1.1.6') CCAPCore.ccap_core_db[second_core.ccap_core_id] = second_core mgr.core_orchestrator.active_list.append( ('eth0', '1.1.1.1', second_core.ccap_core_id)) second_core.agent_status[ProcessAgent.AGENTTYPE_IPSEC] = True mgr.core_orchestrator.seek_principal_core() # principal core not none mgr.principal_core = ccap_core[0] mgr.core_orchestrator.seek_principal_core() mgr.fsm.Error(msg='test fsm error')
def test_core_fail(self): print '*' * 80 print "start to test interface fail" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.interface_candidate.append('eth0') mgr.fsm.STARTUP_DHCP_OK(interface='eth0') mgr.fsm.CORE_FAIL(interface='eth0', core_ip='1.1.1.1', msg='test') mgr.fsm.Error(msg='test fsm error') # provisioning, time not confirmed mgr1 = ManagerProcess(simulator=False) mgr1.fsm.INTERFACE_SCAN(interface='eth0') mgr1.interface_candidate.append('eth0') mgr1.fsm.STARTUP_DHCP_OK(interface='eth0') mgr1.fsm.CORE_FAIL(interface='eth0', core_ip='1.1.1.1', msg='test') mgr1.fsm.Error(msg='test fsm error')
def test_fsm_provision_startup_core_fail_1(self): print '*' * 80 print "start to test fsm_provision_startup_core_fail 1" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.interface_candidate.append('eth0') keys = CCAPCore.ccap_core_db.keys() for key in keys: CCAPCore.ccap_core_db.pop(key) e = _e_obj() event = {"name": 'core_fail', "src": "dhcp", "dst": 'core_fail'} e.fsm, e.event, e.src, e.dst, e.msg, e.interface = mgr.fsm, event, \ 'dhcp', 'core_fail', 'test core fail', 'eth0' # eth0 in candidate mgr._fsm_provision_startup_core_fail(e) mgr.fsm.Error(msg='test fsm error')
def test_basic_fun(self): print '*' * 80 print "start to test basic function" mgr = ManagerProcess(simulator=False) # fd, event mask mgr._handle_agent_event(0, 0) mgr._handle_agent_event(mgr.process_agent_db[ ProcessAgent.AGENTTYPE_INTERFACE_STATUS]['recvSock'].sock, mgr.dispatcher.EV_FD_ERR) # error mgr._handle_agent_event( mgr.process_agent_db[ProcessAgent.AGENTTYPE_INTERFACE_STATUS] ['recvSock'].sock, 10)
def test_gcp_roel(self): print '*' * 80 print "start to test _handle_mgr_gcp_event role" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.fsm.STARTUP_DHCP_OK(interface='eth0') event_request_rsp = protoDef.msg_event_notification() event_request_rsp.mgr_event.mgr_id = mgr.mgr_id event_request_rsp.mgr_event.event_id = ProcessAgent.AGENTTYPE_GCP ccap_core = CCAPCore.ccap_core_db.values() ccap_core[0].interface = 'eth0' ccap_core[0].ccap_core_network_address = '1.1.1.5' # ccap_core[0].is_principal = CoreDescription.CORE_ROLE_PRINCIPAL # ccap_core[0].is_active = CoreDescription.CORE_MODE_ACTIVE # not exist core caps = { "is_active": False, "ccap_core": '1.1.1.10', 'interface': 'eth0', "is_principal": False } event_request_rsp.mgr_event.data = json.dumps("role/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) # principal active caps = { "is_active": True, "ccap_core": '1.1.1.5', 'interface': 'eth0', "is_principal": True } event_request_rsp.mgr_event.data = json.dumps("role/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) # auxiliary standby caps = { "is_active": False, "ccap_core": '1.1.1.5', 'interface': 'eth0', "is_principal": False } event_request_rsp.mgr_event.data = json.dumps("role/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) mgr.fsm.Error(msg='test fsm error')
def test_fsm_provision_core_fail(self): print '*' * 80 print "start to test _fsm_provision_core_fail" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.interface_candidate.append('eth0') e = _e_obj() event = {"name": 'core_fail', "src": "dhcp", "dst": 'core_fail'} e.fsm, e.event, e.src, e.dst, e.msg, e.interface = mgr.fsm, event, \ 'dhcp', 'core_fail', 'test core fail', 'eth0' mgr._fsm_provision_core_fail(e) mgr.fsm.Error(msg='test fsm error') mgr1 = ManagerProcess(simulator=False) mgr1.fsm.INTERFACE_SCAN(interface='eth0') mgr1.interface_candidate.append('eth0') e = _e_obj() event = {"name": 'core_fail', "src": "dhcp", "dst": 'core_fail'} e.fsm, e.event, e.src, e.dst, e.msg, e.interface = mgr1.fsm, event, \ 'dhcp', 'core_fail', 'test core fail', 'eth0' mgr1.set_time(time.time(), step=t_TpcMessage.SUCCESS) mgr1._fsm_provision_core_fail(e) mgr1.fsm.Error(msg='test fsm error')
def test_gcp_Ha(self): print '*' * 80 print "start to test _handle_mgr_gcp_event HA" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.fsm.STARTUP_DHCP_OK(interface='eth0') event_request_rsp = protoDef.msg_event_notification() event_request_rsp.mgr_event.mgr_id = mgr.mgr_id event_request_rsp.mgr_event.event_id = ProcessAgent.AGENTTYPE_GCP ccap_core = CCAPCore.ccap_core_db.values() ccap_core[0].interface = 'eth0' ccap_core[0].ccap_core_network_address = '1.1.1.5' print 'Redundant core Add' # invalid ip caps = { "ActiveCoreIpAddress": '0.0.0.0', "StandbyCoreIpAddress": '1.1.1.10', 'interface': 'eth0', "operation": 0 } event_request_rsp.mgr_event.data = json.dumps("Ha/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) # not exist core caps = { "ActiveCoreIpAddress": '1.1.1.10', "StandbyCoreIpAddress": '1.1.1.11', 'interface': 'eth0', "operation": 0 } event_request_rsp.mgr_event.data = json.dumps("Ha/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) # active core mode active caps = { "ActiveCoreIpAddress": '1.1.1.5', "StandbyCoreIpAddress": '1.1.1.11', 'interface': 'eth0', "operation": 0 } ccap_core[0].is_active = CoreDescription.CORE_MODE_ACTIVE event_request_rsp.mgr_event.data = json.dumps("Ha/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) # active core mode standby caps = { "ActiveCoreIpAddress": '1.1.1.5', "StandbyCoreIpAddress": '1.1.1.11', 'interface': 'eth0', "operation": 0 } ccap_core[0].is_active = CoreDescription.CORE_MODE_STANDBY event_request_rsp.mgr_event.data = json.dumps("Ha/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) # standby core mode active caps = { "ActiveCoreIpAddress": '1.1.1.10', "StandbyCoreIpAddress": '1.1.1.5', 'interface': 'eth0', "operation": 0 } ccap_core[0].is_active = CoreDescription.CORE_MODE_ACTIVE event_request_rsp.mgr_event.data = json.dumps("Ha/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) # standby core mode standby caps = { "ActiveCoreIpAddress": '1.1.1.10', "StandbyCoreIpAddress": '1.1.1.5', 'interface': 'eth0', "operation": 0 } ccap_core[0].is_active = CoreDescription.CORE_MODE_STANDBY mgr.add_ip_to_core_map('eth0', ('1.1.1.5', '', 0)) event_request_rsp.mgr_event.data = json.dumps("Ha/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) print 'Redundant core Change' ccap_core[0].is_principal = CoreDescription.CORE_ROLE_PRINCIPAL # standby core mode active caps = { "ActiveCoreIpAddress": '1.1.1.10', "StandbyCoreIpAddress": '1.1.1.5', 'interface': 'eth0', "operation": 2 } ccap_core[0].is_active = CoreDescription.CORE_MODE_ACTIVE event_request_rsp.mgr_event.data = json.dumps("Ha/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) # standby core mode standby caps = { "ActiveCoreIpAddress": '1.1.1.10', "StandbyCoreIpAddress": '1.1.1.5', 'interface': 'eth0', "operation": 2 } ccap_core[0].is_active = CoreDescription.CORE_MODE_STANDBY event_request_rsp.mgr_event.data = json.dumps("Ha/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) print 'Redundant core Delete' # standby core mode active caps = { "ActiveCoreIpAddress": '1.1.1.10', "StandbyCoreIpAddress": '1.1.1.5', 'interface': 'eth0', "operation": 1 } ccap_core[0].is_active = CoreDescription.CORE_MODE_ACTIVE event_request_rsp.mgr_event.data = json.dumps("Ha/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) # standby core mode standby caps = { "ActiveCoreIpAddress": '1.1.1.10', "StandbyCoreIpAddress": '1.1.1.5', 'interface': 'eth0', "operation": 1 } ccap_core[0].is_active = CoreDescription.CORE_MODE_STANDBY event_request_rsp.mgr_event.data = json.dumps("Ha/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) mgr.fsm.Error(msg='test fsm error')
def test_config_table(self): print '*' * 80 print "start to test _handle_mgr_gcp_event config_table" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.fsm.STARTUP_DHCP_OK(interface='eth0') event_request_rsp = protoDef.msg_event_notification() event_request_rsp.mgr_event.mgr_id = mgr.mgr_id event_request_rsp.mgr_event.event_id = ProcessAgent.AGENTTYPE_GCP # invalid ip caps = {"ccap_core": '0.0.0.0', 'interface': 'eth0', "operation": 0} event_request_rsp.mgr_event.data = json.dumps("config_table/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) # add existed ip ccap_core = CCAPCore.ccap_core_db.values() ccap_core[0].interface = 'eth0' ccap_core[0].ccap_core_network_address = '1.1.1.5' ccap_core[0].is_principal = CoreDescription.CORE_ROLE_PRINCIPAL ccap_core[0].is_active = CoreDescription.CORE_MODE_ACTIVE # CCAPCore('CORE-1234567890', CoreDescription.CORE_ROLE_PRINCIPAL, # CoreDescription.CORE_MODE_ACTIVE, mgr=mgr, # ccap_core_interface='etho0', ccap_core_network_address='1.1.1.5') caps = { "ccap_core": '1.1.1.5', 'interface': 'eth0', "operation": 0 } # add event_request_rsp.mgr_event.data = json.dumps("config_table/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) # add a new one caps = { "ccap_core": '1.1.1.6', 'interface': 'eth0', "operation": 0 } # add event_request_rsp.mgr_event.data = json.dumps("config_table/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) caps = { "ccap_core": '1.1.1.5', 'interface': 'eth0', "operation": 2 } # change event_request_rsp.mgr_event.data = json.dumps("config_table/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) mgr.add_ip_to_core_map('eth0', ('1.1.1.5', '', 4)) caps = { "ccap_core": '1.1.1.5', 'interface': 'eth0', "operation": 1 } # delete event_request_rsp.mgr_event.data = json.dumps("config_table/" + json.dumps(caps)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) mgr.fsm.Error(msg='test fsm error')
def setUpClass(cls): setup_logging("PROVISION", "test.log") cls.mgr = ManagerProcess(simulator=True, test_flag=True)
def test_handle_mgr_gcp_event(self): print '*' * 80 print "start to test _handle_mgr_gcp_event" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.fsm.STARTUP_DHCP_OK(interface='eth0') event_request_rsp = protoDef.msg_event_notification() event_request_rsp.mgr_event.mgr_id = mgr.mgr_id event_request_rsp.mgr_event.event_id = ProcessAgent.AGENTTYPE_GCP # interface set to '' event_request_rsp.mgr_event.data = json.dumps("reboot/;1.1.1.1;info") mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) event_request_rsp.mgr_event.data = json.dumps( "connect_closed/;1.1.1.1;" + str(True)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) event_request_rsp.mgr_event.data = json.dumps( "gcp_flapping/;1.1.1.1/recovering") mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) event_request_rsp.mgr_event.data = json.dumps( "redirect/1.1.1.3;/;1.1.1.1/") mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) # interface and ip valid ccap_core = CCAPCore.ccap_core_db.values() ccap_core[0].interface = 'eth0' ccap_core[0].ccap_core_network_address = '1.1.1.5' mgr.add_ip_to_core_map('eth0', ('1.1.1.5', '', 4)) mgr.principal_core = ccap_core[0] second_core = CCAPCore('CORE-1234567890', CoreDescription.CORE_ROLE_PRINCIPAL, CoreDescription.CORE_MODE_ACTIVE, mgr=mgr, ccap_core_interface='eth0', ccap_core_network_address='1.1.1.6') CCAPCore.ccap_core_db[second_core.ccap_core_id] = second_core event_request_rsp.mgr_event.data = json.dumps( "reboot/eth0;1.1.1.5;info") mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) ccap_core = CCAPCore('CORE-1234567890', CoreDescription.CORE_ROLE_PRINCIPAL, CoreDescription.CORE_MODE_ACTIVE, mgr=mgr, ccap_core_interface='eth0', ccap_core_network_address='1.1.1.5') CCAPCore.ccap_core_db[ccap_core.ccap_core_id] = ccap_core event_request_rsp.mgr_event.data = json.dumps( "connect_closed/eth0;1.1.1.5;" + str(True)) mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) event_request_rsp.mgr_event.data = json.dumps( "gcp_flapping/eth0;1.1.1.5/recovering") mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) event_request_rsp.mgr_event.data = json.dumps( "gcp_flapping/eth0;1.1.1.5/done") mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) mgr.add_ip_to_core_map('eth0', ('1.1.1.5', '', 4)) event_request_rsp.mgr_event.data = json.dumps( "redirect/1.1.1.3;/eth0;1.1.1.5") mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) mgr.add_ip_to_core_map('eth0', ('1.1.1.5', '', 4)) event_request_rsp.mgr_event.data = json.dumps( "get_active_principal/3,1.1.1.1/") mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event)
def test_seek_principal_failure_3(self): print '*' * 80 print "start to test seek_principal_failure 3" mgr3 = ManagerProcess(simulator=False) mgr3.fsm.INTERFACE_SCAN(interface='eth0') mgr3.fsm.INTERFACE_SCAN(interface='eth0') mgr3.fsm.STARTUP_DHCP_OK(interface='eth0') mgr3.set_time(time.time(), step=t_TpcMessage.SUCCESS) mgr3._principal_core_seek_failure_before_tod(None) mgr3.set_time(time.time(), step=t_TpcMessage.FIRST_ATTEMPT_FAILED) mgr3._principal_core_seek_failure_before_tod(None) # .principal_core_seek_timer not none mgr3.set_time(time.time(), step=t_TpcMessage.FIRST_ATTEMPT_FAILED) mgr3._principal_core_seek_failure_before_tod(None) mgr3.fsm.Error(msg='test fsm error')
def test_core_map(self): print '*' * 80 print "start to test interface fail" mgr = ManagerProcess(simulator=False) mgr.is_ip_in_core_map('eth0', '1.1.1.1') mgr.remove_ip_in_core_map('eth0', '1.1.1.1') mgr.add_ip_to_core_map('eth0', ('1.1.1.1', '', 4)) mgr.get_core_map('eth0', '1.1.1.1') mgr.get_core_map('eth0', '1.1.1.1') mgr.is_valid_ip('1.1')
def test_handle_mgr_tod_event(self): print '*' * 80 print "start to test _handle_mgr_tod_event" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.fsm.STARTUP_DHCP_OK(interface='eth0') event_request_rsp = protoDef.msg_event_notification() event_request_rsp.mgr_event.mgr_id = mgr.mgr_id event_request_rsp.mgr_event.event_id = ProcessAgent.AGENTTYPE_TOD event_request_rsp.mgr_event.data = 'tod_first_failed/0' mgr._handle_mgr_tod_event(event_request_rsp.mgr_event) event_request_rsp.mgr_event.data = 'tod_failed/' mgr._handle_mgr_tod_event(event_request_rsp.mgr_event) event_request_rsp.mgr_event.data = 'tod_failed/' mgr.tod_retry = mgr.TOD_RETRY_CNT mgr._handle_mgr_tod_event(event_request_rsp.mgr_event) event_request_rsp.mgr_event.data = 'else/' mgr._handle_mgr_tod_event(event_request_rsp.mgr_event) mgr.fsm.Error(msg='test fsm error')
def test_handle_mgr_dhcp_event(self): print '*' * 80 print "start to test _handle_mgr_dhcp_event" mgr = ManagerProcess(simulator=False) mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.fsm.STARTUP_DHCP_OK(interface='eth0') event_request_rsp = protoDef.msg_event_notification() event_request_rsp.mgr_event.mgr_id = mgr.mgr_id event_request_rsp.mgr_event.event_id = ProcessAgent.AGENTTYPE_DHCP ccap_core = CCAPCore.ccap_core_db.values() ccap_core[0].interface = 'eth0' # no data field mgr._handle_mgr_dhcp_event(event_request_rsp.mgr_event) # with core list event_request_rsp.mgr_event.data = json.dumps({ 'CCAPCores': [ '1.1.1.1', ], 'TimeServers': [ '1.1.1.1', ], 'TimeOffset': 100, 'LogServers': [ '1.1.1.1', ], 'initiated_by': None, 'Interface': 'eth0' }) mgr._handle_mgr_dhcp_event(event_request_rsp.mgr_event) # eth0 already in dhcp_parameters, update it event_request_rsp.mgr_event.data = json.dumps({ 'CCAPCores': ['1.1.1.1', '1.1.1.7'], 'TimeServers': [ '1.1.1.1', ], 'TimeOffset': 100, 'LogServers': [ '1.1.1.1', ], 'initiated_by': None, 'Interface': 'eth0' }) mgr._handle_mgr_dhcp_event(event_request_rsp.mgr_event) # no update event_request_rsp.mgr_event.data = json.dumps({ 'CCAPCores': ['1.1.1.1', '1.1.1.7'], 'TimeServers': [ '1.1.1.1', ], 'TimeOffset': 100, 'LogServers': [ '1.1.1.1', ], 'initiated_by': None, 'Interface': 'eth0' }) mgr._handle_mgr_dhcp_event(event_request_rsp.mgr_event) # no core list event_request_rsp.mgr_event.data = json.dumps({ 'CCAPCores': [], 'TimeServers': [ '1.1.1.1', ], 'TimeOffset': 100, 'LogServers': [ '1.1.1.1', ], 'initiated_by': None, 'Interface': 'eth0' }) mgr._handle_mgr_dhcp_event(event_request_rsp.mgr_event) mgr.fsm.Error(msg='test fsm error')
def test_start_process(self): """Start provision process with real agent.""" print '*' * 80 print "start to test provision manager with real agent" mgr = ManagerProcess(simulator=False) mgr.dhcp_parameter['eth0'] = { 'TimeServers': [ '1.1.1.1', ], 'TimeOffset': 100, 'CCAPCores': [ '1.1.1.1', ], 'LogServers': [ '1.1.1.1', ] } # no core mgr.core_orchestrator.orchestrator_cb(None) # startup mgr.fsm.INTERFACE_SCAN(interface='eth0') mgr.add_ip_to_core_map('eth0', ('1.1.1.1', '', mgr.HA_CORE_TRIGGER)) mgr.core_orchestrator.orchestrator_cb(None) mgr.remove_ip_in_core_map('eth0', ('1.1.1.1', '', mgr.HA_CORE_TRIGGER)) # cb w/o candidate list mgr.core_orchestrator.orchestrator_cb(None) mgr.add_ip_to_core_map('eth0', ('1.1.1.1', '', mgr.HA_CORE_TRIGGER)) mgr.interface_list.append({ "interface": 'eth0', "status": mgr.INTERFACE_UP, }) mgr.core_orchestrator.orchestrator_cb(None) # is system time confirmed. mgr.set_time(time.time()) mgr.is_system_time_confirmed() # mgr state transmit mgr.fsm.STARTUP_DHCP_OK() mgr.core_orchestrator.orchestrator_cb(None) # operational mgr.fsm.OPERATIONAL_OK() print mgr.fsm.current, mgr.core_orchestrator.active_list mgr.core_orchestrator.orchestrator_cb(None) # check ka mgr.check_ka_status(None) mgr.check_ka_status(None) mgr.check_ka_status(None) keys = CCAPCore.ccap_core_db.keys() for key in keys: CCAPCore.ccap_core_db.pop(key) mgr.process_agent_db[ProcessAgent.AGENTTYPE_INTERFACE_STATUS][ "status"] = mgr.MGR_UNREGISTERED mgr.check_ka_status(None) # fail mgr.core_orchestrator.orchestrator_cb(None) mgr.fsm.Error(msg='test fsm error')
class testManagerProcess(unittest.TestCase): __metaclass__ = AddLoggerToClass @classmethod def setUpClass(cls): setup_logging("PROVISION", "test_mgr_process.log") cls.fake_cb_cnt = 0 @classmethod def tearDownClass(cls): pass def setUp(self): setup_test_redis() CCAPCoreOrchestrator.NO_PRINCIPAL_CORE_FOUND_TIMEOUT = 60 CCAPCore.ccap_core_db = {} self.fake_cb_cnt = 0 self.mgr = ManagerProcess(simulator=True, test_flag=True) ManagerProcess.SYSTEM_TIME_CONFIRM = "None" def tearDown(self): CCAPCoreOrchestrator.NO_PRINCIPAL_CORE_FOUND_TIMEOUT = 60 CCAPCore.ccap_core_db = {} ManagerProcess.SYSTEM_TIME_CONFIRM = "None" stop_test_redis() def fake_cb(self, _): self.fake_cb_cnt += 1 return def test_is_system_time_confirmed(self): backup = ManagerProcess.SYSTEM_TIME_CONFIRM ManagerProcess.SYSTEM_TIME_CONFIRM = t_TpcMessage.INITIATED result = ManagerProcess.is_system_time_confirmed() self.assertFalse(result) ManagerProcess.SYSTEM_TIME_CONFIRM = t_TpcMessage.SUCCESS result = ManagerProcess.is_system_time_confirmed() self.assertTrue(result) ManagerProcess.SYSTEM_TIME_CONFIRM = backup self.assertEquals(ManagerProcess.SYSTEM_TIME_CONFIRM, "None") def test_system_time_confirmed(self): backup = ManagerProcess.SYSTEM_TIME_CONFIRM ManagerProcess._system_time_confirmed(step=t_TpcMessage.SUCCESS) result = ManagerProcess.is_system_time_confirmed() self.assertTrue(result) ManagerProcess.SYSTEM_TIME_CONFIRM = backup self.assertEquals(ManagerProcess.SYSTEM_TIME_CONFIRM, "None") def test_set_time(self): try: self.mgr.set_time(timestamp=112314) except Exception as e: self.fail("Exception happened : %s" % e) def test_system_operational_timeout(self): self.mgr._system_operational_timeout("unittest") self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_FAIL) def test_start_principal_core_seek_timer(self): old_timeout = ManagerProcess.SEEK_PRINCIPAL_CORE_TIMEOUT self.assertIsNone(self.mgr.principal_core_seek_timer) ManagerProcess.SEEK_PRINCIPAL_CORE_TIMEOUT = 100 self.mgr.start_principal_core_seek_timer(self.fake_cb) self.assertIsNotNone(self.mgr.principal_core_seek_timer) ManagerProcess.SEEK_PRINCIPAL_CORE_TIMEOUT = 1 self.mgr.start_principal_core_seek_timer(self.fake_cb) self.assertIsNotNone(self.mgr.principal_core_seek_timer) time.sleep(1) self.mgr.dispatcher.handle_one_event() self.assertEqual(self.fake_cb_cnt, 1) ManagerProcess.SEEK_PRINCIPAL_CORE_TIMEOUT = old_timeout self.assertEquals(ManagerProcess.SEEK_PRINCIPAL_CORE_TIMEOUT, old_timeout) def test_create_original_core(self): self.mgr.create_original_core(interface="eth0") self.assertEquals(len(CCAPCore.ccap_core_db), 1) for core in CCAPCore.ccap_core_db.values(): self.assertIsInstance(core, CCAPCore) self.assertIsInstance(core.fsm, CCAPFsmStartup) self.assertEquals(core.initiated_by, "Startup") def test_fsm_provision_interface_scan(self): try: self.mgr.fsm.INTERFACE_SCAN(interface="eth0") self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_INTERFACE_PROVISION) except Exception as e: self.fail("Exception happened : %s" % e) def test_principal_core_seek_failure(self): self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_INIT) self.mgr.fsm.INTERFACE_SCAN(interface="eth0") self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_INTERFACE_PROVISION) self.mgr.fsm.STARTUP_DHCP_OK() self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPLE_PROVISION) self.mgr.principal_core_seek_failure("unittest") self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPLE_RETRY_FIRST) def test_fsm_provision_state_retry(self): self.mgr.fsm.fsm.current = self.mgr.fsm.STATE_PRINCIPLE_RETRY_FIRST self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPLE_RETRY_FIRST) print "Start test fsm_provision_state_retry" print "Check equal state" self.mgr.fsm.ENTER_CURRENT_STATE() self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPLE_RETRY_FIRST) print "Check Principal present" self.mgr.principal_core = "unittest" self.mgr.core_orchestrator.active_list.append('1.1.1.1') self.mgr.principal_core_seek_failure("unittest") self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPLE_RETRY_SECOND) self.assertEquals(len(self.mgr.core_orchestrator.active_list), 1) print "Check reset active list and failed list" self.mgr.principal_core = None self.mgr.principal_core_seek_failure("unittest") self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPLE_RETRY_THIRD) self.assertEquals(len(self.mgr.core_orchestrator.active_list), 0) self.assertEquals(len(self.mgr.core_orchestrator.failed_list), 0) def test_principal_core_seek_failure_before_tod(self): print "Check system first failed " self.assertIsNone(self.mgr.principal_core_seek_timer) ManagerProcess.SYSTEM_TIME_CONFIRM = t_TpcMessage.FIRST_ATTEMPT_FAILED self.mgr._principal_core_seek_failure_before_tod("unittest") self.assertIsNotNone(self.mgr.principal_core_seek_timer) print "Check system tod initiated " ManagerProcess.SYSTEM_TIME_CONFIRM = t_TpcMessage.INITIATED self.mgr._principal_core_seek_failure_before_tod("unittest") self.assertIsNotNone(self.mgr.principal_core_seek_timer) print "Check system time is confirmed" ManagerProcess.SYSTEM_TIME_CONFIRM = t_TpcMessage.SUCCESS self.mgr._principal_core_seek_failure_before_tod("unittest") self.assertIsNone(self.mgr.principal_core_seek_timer) print "Check system tod failed " ManagerProcess.SYSTEM_TIME_CONFIRM = t_TpcMessage.FIRST_ATTEMPT_FAILED self.mgr._principal_core_seek_failure_before_tod("unittest") self.assertIsNotNone(self.mgr.principal_core_seek_timer) ManagerProcess.SYSTEM_TIME_CONFIRM = t_TpcMessage.ALL_ATTEMPTS_FAILED self.mgr._principal_core_seek_failure_before_tod("unittest") self.assertIsNone(self.mgr.principal_core_seek_timer) def test_fsm_provision_startup(self): self.assertIsNotNone(self.mgr.interface_scan_timer) def test_fsm_provision_user_mgmt(self): try: self.mgr.fsm.USER_MGMT() except Exception as e: self.fail("Exception happened : %s" % e) def test_fsm_provision_gcp_mgmt(self): try: self.mgr.fsm.GCP_MGMT() except Exception as e: self.fail("Exception happened : %s" % e) def test_fsm_provision_dhcp(self): try: self.mgr.fsm.DHCP() except Exception as e: self.fail("Exception happened : %s" % e) def test_fsm_provision_principal_core_found(self): print "Check state no change" old = self.mgr.fsm.current self.mgr.fsm.SEEK_PRINCIPAL_OK() self.assertEquals(self.mgr.fsm.current, old) print "Check state change to principal found" self.mgr.fsm.fsm.current = self.mgr.fsm.STATE_PRINCIPLE_PROVISION self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPLE_PROVISION) self.mgr.principal_core_seek_timer = self.mgr.dispatcher.timer_register( self.mgr.SEEK_PRINCIPAL_CORE_TIMEOUT, self.fake_cb) self.assertIsNotNone(self.mgr.principal_core_seek_timer) self.mgr.fsm.SEEK_PRINCIPAL_OK() self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPAL_FOUND) self.assertIsNone(self.mgr.principal_core_seek_timer) def set_mgr_ready_for_8021x(self): print "Interface UP" self.mgr.interface_up_handler(up_interface_lists=['eth0', 'eth1']) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_INTERFACE_PROVISION) self.assertIsNone(self.mgr.interface_scan_timer) self.assertEquals(len(CCAPCore.ccap_core_db), 1) for startupcore in CCAPCore.ccap_core_db.values(): self.assertIsInstance(startupcore, CCAPCore) self.assertIsInstance(startupcore.fsm, CCAPFsmStartup) self.assertEquals(startupcore.initiated_by, "Startup") self.assertEquals(startupcore.fsm.current, "none") break startupcore.fsm.TRIGGER_Startup() self.assertEquals(startupcore.fsm.current, startupcore.fsm.STATE_INIT) msg = TestCCAPCoreClass.build_event_notification( ccap_id=startupcore.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="UP", agentid=provision_pb2.AGENTTYPE_INTERFACE_STATUS) startupcore._handle_core_interface_event(msg) self.assertEquals(startupcore.fsm.current, startupcore.fsm.STATE_INTERFACE_UP) return True def set_mgr_ready_for_dhcp(self): self.assertTrue(self.set_mgr_ready_for_8021x()) startupcore = self.get_startup_core() msg = TestCCAPCoreClass.build_event_notification( ccap_id=startupcore.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="UP", agentid=provision_pb2.AGENTTYPE_8021X) startupcore._handle_core_8021x_event(msg) self.assertEqual(startupcore.fsm.current, CCAPFsmStartup.STATE_8021X_OK) return True def set_mgr_ready_for_tod(self): self.assertTrue(self.set_mgr_ready_for_dhcp()) startupcore = self.get_startup_core() print "Manager DHCP OK" msg = TestCCAPCoreClass.build_event_notification( ccap_id=startupcore.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="UP", agentid=provision_pb2.AGENTTYPE_DHCP) startupcore._handle_core_dhcp_event(msg) self.assertEqual(startupcore.fsm.current, CCAPFsmStartup.STATE_DHCP_OK) event_request_rsp = process_agent_pb2.msg_event_notification() event_request_rsp.mgr_event.mgr_id = self.mgr.mgr_id event_request_rsp.mgr_event.event_id = provision_pb2.AGENTTYPE_DHCP event_request_rsp.mgr_event.data = \ json.dumps( {'CCAPCores': ['1.1.1.1', '2.2.2.2'], 'TimeServers': ['1.1.1.1', '2.2.2.2'], 'TimeOffset': 12800, 'CreatedTime': 22222, 'LogServers': ['1.1.1.1', '2.2.2.2'], 'initiated_by': startupcore.ccap_core_id, 'Interface': startupcore.interface}) self.mgr._handle_mgr_dhcp_event(msg=event_request_rsp.mgr_event) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPLE_PROVISION) return True def set_mgr_ready_for_ipsec(self, tod_server="1.1.1.1", timestamp="15234215315"): self.assertTrue(self.set_mgr_ready_for_tod()) print "Manager and startup core TOD OK" startupcore = self.get_startup_core() startupcore.fsm.TRIGGER_TOD_OK() self.assertEquals(startupcore.fsm.current, startupcore.fsm.STATE_TOD_OK) event_request_rsp = process_agent_pb2.msg_event_notification() event_request_rsp.mgr_event.mgr_id = self.mgr.mgr_id event_request_rsp.mgr_event.event_id = provision_pb2.AGENTTYPE_TOD event_request_rsp.mgr_event.data = 'success/' + timestamp + \ '|' + tod_server self.mgr._handle_mgr_tod_event(msg=event_request_rsp.mgr_event) self.assertTrue(self.mgr.is_system_time_confirmed()) self.assertIsNotNone(self.mgr.operational_timer) self.assertIsNotNone(self.mgr.core_orchestrator.orchestrator_timer) self.assertIsNone(self.mgr.principal_core_seek_timer) self.assertEquals(len(CCAPCore.ccap_core_db), 2) for gcpcore in CCAPCore.ccap_core_db.values(): if isinstance(gcpcore, CCAPCore) and isinstance( gcpcore.fsm, CCAPFsm): break self.assertIsInstance(gcpcore, CCAPCore) print "GCP core startup OK" gcpcore.fsm.TRIGGER_Startup() self.assertEquals(gcpcore.fsm.current, gcpcore.fsm.STATE_INIT_IPSEC) return True def set_core_ready_for_tcp(self, coreid): print "GCP core IPsec OK" gcpcore = CCAPCore.ccap_core_db[coreid] self.assertEquals(gcpcore.fsm.current, gcpcore.fsm.STATE_INIT_IPSEC) self.receive_ipsec_ok(gcpcore.ccap_core_id) self.assertEquals(gcpcore.fsm.current, gcpcore.fsm.STATE_INIT_TCP) return True def set_mgr_principal_found(self, core_ip='1.1.1.1'): self.assertTrue(self.set_mgr_ready_for_ipsec()) for gcpcore in CCAPCore.ccap_core_db.values(): if isinstance(gcpcore, CCAPCore) and isinstance(gcpcore.fsm, CCAPFsm) \ and (gcpcore.ccap_core_network_address == core_ip): break try: self.set_core_ready_for_tcp(coreid=gcpcore.ccap_core_id) print "Manager receive core identification" caps = { "index": 1, "is_active": True, "ccap_core": gcpcore.ccap_core_network_address, 'interface': gcpcore.interface, "is_principal": True } event_request_rsp = process_agent_pb2.msg_event_notification() event_request_rsp.mgr_event.mgr_id = self.mgr.mgr_id event_request_rsp.mgr_event.event_id = provision_pb2.AGENTTYPE_GCP event_request_rsp.mgr_event.data = json.dumps("role/" + json.dumps(caps)) self.mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPAL_FOUND) self.assertIsNone(self.mgr.principal_core_seek_timer) self.assertIsNotNone(self.mgr.principal_core) return True except Exception as e: raise e def recv_gcp_msg(self, coreid, msg): gcpcore = CCAPCore.ccap_core_db[coreid] msg = TestCCAPCoreClass.build_event_notification( ccap_id=gcpcore.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result=msg, agentid=provision_pb2.AGENTTYPE_GCP) gcpcore._handle_core_gcp_event(msg) return True def receive_ipsec_ok(self, coreid): gcpcore = CCAPCore.ccap_core_db[coreid] msg = TestCCAPCoreClass.build_event_notification( ccap_id=coreid, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="UP", agentid=provision_pb2.AGENTTYPE_IPSEC) gcpcore._handle_core_ipsec_event(msg) return True def set_core_op(self, coreid): gcpcore = CCAPCore.ccap_core_db[coreid] print "Gcp core receive msg" self.assertTrue(self.recv_gcp_msg(coreid, "TCP_OK")) self.assertEqual(gcpcore.fsm.current, CCAPFsm.STATE_INIT_GCP_IRA) self.assertTrue(self.recv_gcp_msg(coreid, "GCP_IRA")) self.assertEqual(gcpcore.fsm.current, CCAPFsm.STATE_INIT_GCP_CFG) self.assertTrue(self.recv_gcp_msg(coreid, "GCP_CFG")) self.assertEqual(gcpcore.fsm.current, CCAPFsm.STATE_INIT_GCP_CFG_CPL) self.assertTrue(self.recv_gcp_msg(coreid, "GCP_CFG_CPL")) self.assertEqual(gcpcore.fsm.current, CCAPFsm.STATE_INIT_GCP_OP) self.assertTrue(self.recv_gcp_msg(coreid, "OPERATIONAL")) self.assertEqual(gcpcore.fsm.current, CCAPFsm.STATE_ONLINE) return True def set_mgr_operational(self): self.assertTrue(self.set_mgr_principal_found()) self.set_core_op(self.mgr.principal_core.ccap_core_id) self.assertEqual(self.mgr.fsm.current, self.mgr.fsm.STATE_OPERATIONAL) self.assertIsNone(self.mgr.operational_timer) # trigger gcp core to startup gcpcore = self.get_core("eth0", '2.2.2.2') self.assertIsNotNone(gcpcore) gcpcore.fsm.TRIGGER_Startup() self.assertEqual(gcpcore.fsm.current, gcpcore.fsm.STATE_INIT_IPSEC) return True def get_startup_core(self): return self.mgr.startup_core def test_scenario_from_init_to_operational(self): self.assertTrue(self.set_mgr_operational()) def test_scenario_from_init_to_todfail(self): self.assertTrue(self.set_mgr_ready_for_tod()) startupcore = self.get_startup_core() print "First tod fail" event_request_rsp = process_agent_pb2.msg_event_notification() event_request_rsp.mgr_event.mgr_id = self.mgr.mgr_id event_request_rsp.mgr_event.event_id = provision_pb2.AGENTTYPE_TOD event_request_rsp.mgr_event.data = 'tod_first_failed/0' + '|' + '1.1.1.1' self.mgr._handle_mgr_tod_event(msg=event_request_rsp.mgr_event) self.assertEquals(ManagerProcess.SYSTEM_TIME_CONFIRM, t_TpcMessage.FIRST_ATTEMPT_FAILED) print "All tod fail" event_request_rsp = process_agent_pb2.msg_event_notification() event_request_rsp.mgr_event.mgr_id = self.mgr.mgr_id event_request_rsp.mgr_event.event_id = provision_pb2.AGENTTYPE_TOD event_request_rsp.mgr_event.data = 'tod_failed/' + '' + '|' + '1.1.1.1' self.mgr._handle_mgr_tod_event(msg=event_request_rsp.mgr_event) self.assertEquals(self.mgr.tod_retry, 1) self.assertEquals(ManagerProcess.SYSTEM_TIME_CONFIRM, t_TpcMessage.ALL_ATTEMPTS_FAILED) print "Second tod fail" event_request_rsp = process_agent_pb2.msg_event_notification() event_request_rsp.mgr_event.mgr_id = self.mgr.mgr_id event_request_rsp.mgr_event.event_id = provision_pb2.AGENTTYPE_TOD event_request_rsp.mgr_event.data = 'tod_failed/' + '' + '|' + '1.1.1.1' self.mgr._handle_mgr_tod_event(msg=event_request_rsp.mgr_event) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_FAIL) def set_mgr_all_core_fail(self): for gcpcore in CCAPCore.ccap_core_db.values(): if isinstance(gcpcore, CCAPCore) and isinstance( gcpcore.fsm, CCAPFsm): break gcpcore.fsm.TRIGGER_Error() self.assertEquals(len(self.mgr.core_orchestrator.failed_list), 1) self.assertEquals(len(CCAPCore.ccap_core_db), 2) for gcpcore in CCAPCore.ccap_core_db.values(): if isinstance(gcpcore, CCAPCore) and isinstance( gcpcore.fsm, CCAPFsm): break self.assertEquals(gcpcore.ccap_core_network_address, '2.2.2.2') CCAPCoreOrchestrator.NO_PRINCIPAL_CORE_FOUND_TIMEOUT = 1 gcpcore.fsm.TRIGGER_Error() self.assertEquals(len(self.mgr.core_orchestrator.failed_list), 2) self.assertIsNotNone(self.mgr.principal_core_seek_timer) time.sleep(1) self.mgr.dispatcher.handle_one_event() return True def test_scenario_from_init_to_principal_not_found(self): self.assertTrue(self.set_mgr_ready_for_ipsec()) for gcpcore in CCAPCore.ccap_core_db.values(): if isinstance(gcpcore, CCAPCore) and isinstance( gcpcore.fsm, CCAPFsm): break try: print "Core fail" self.assertTrue(self.set_mgr_all_core_fail()) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPLE_RETRY_FIRST) print "First retry" self.assertTrue(self.set_mgr_all_core_fail()) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPLE_RETRY_SECOND) print "Second retry" self.assertTrue(self.set_mgr_all_core_fail()) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPLE_RETRY_THIRD) print "Third retry" self.assertTrue(self.set_mgr_all_core_fail()) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_FAIL) except Exception as e: self.fail("Exception happened : %s" % e) def test_scenario_from_op_to_principal_gcp_reconnect_ok(self): self.assertTrue(self.set_mgr_operational()) principal_core = self.mgr.principal_core msg = TestCCAPCoreClass.build_event_notification( ccap_id=principal_core.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="TCP_FAIL", agentid=provision_pb2.AGENTTYPE_GCP) principal_core._handle_core_gcp_event(msg) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_REINIT_IPSEC) self.assertIsNotNone( principal_core.registered_timers[provision_pb2.AGENTTYPE_IPSEC]) self.assertEquals( principal_core.state_retried_times[ principal_core.fsm.STATE_REINIT_IPSEC], 1) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPAL_FOUND) self.assertFalse( principal_core.agent_status[provision_pb2.AGENTTYPE_IPSEC]) msg = TestCCAPCoreClass.build_event_notification( ccap_id=principal_core.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="UP", agentid=provision_pb2.AGENTTYPE_IPSEC) principal_core._handle_core_ipsec_event(msg) self.assertTrue( principal_core.agent_status[provision_pb2.AGENTTYPE_IPSEC]) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_REINIT_TCP) msg = TestCCAPCoreClass.build_event_notification( ccap_id=principal_core.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="TCP_FAIL", agentid=provision_pb2.AGENTTYPE_GCP) principal_core._handle_core_gcp_event(msg) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_REINIT_IPSEC) self.assertIsNotNone( principal_core.registered_timers[provision_pb2.AGENTTYPE_IPSEC]) self.assertEquals( principal_core.state_retried_times[ principal_core.fsm.STATE_REINIT_IPSEC], 2) self.assertFalse( principal_core.action_status[provision_pb2.AGENTTYPE_IPSEC]) msg = TestCCAPCoreClass.build_event_notification( ccap_id=principal_core.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="UP", agentid=provision_pb2.AGENTTYPE_IPSEC) principal_core._handle_core_ipsec_event(msg) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_REINIT_TCP) msg = TestCCAPCoreClass.build_event_notification( ccap_id=principal_core.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="TCP_OK", agentid=provision_pb2.AGENTTYPE_GCP) principal_core._handle_core_gcp_event(msg) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_REINIT_GCP_IRA) self.assertEquals( principal_core.state_retried_times[ principal_core.fsm.STATE_REINIT_IPSEC], 2) msg = TestCCAPCoreClass.build_event_notification( ccap_id=principal_core.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="GCP_IRA", agentid=provision_pb2.AGENTTYPE_GCP) principal_core._handle_core_gcp_event(msg) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_ONLINE) self.assertEquals( principal_core.state_retried_times[ principal_core.fsm.STATE_REINIT_IPSEC], 0) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_OPERATIONAL) def test_scenario_from_op_to_principal_gcp_reconnect_fail(self): self.assertTrue(self.set_mgr_operational()) principal_core = self.mgr.principal_core self.assertEqual( principal_core.state_retried_times[ principal_core.fsm.STATE_REINIT_IPSEC], 0) self.assertTrue( self.recv_gcp_msg(principal_core.ccap_core_id, "TCP_FAIL")) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_REINIT_IPSEC) self.assertIsNotNone( principal_core.registered_timers[provision_pb2.AGENTTYPE_IPSEC]) self.assertEquals( principal_core.state_retried_times[ principal_core.fsm.STATE_REINIT_IPSEC], 1) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPAL_FOUND) self.assertFalse( principal_core.agent_status[provision_pb2.AGENTTYPE_IPSEC]) self.assertTrue( principal_core.action_status[provision_pb2.AGENTTYPE_IPSEC]) for i in range( 1, principal_core.CoreStateFailureRetry[ principal_core.fsm.STATE_REINIT_IPSEC] + 1): self.assertTrue(self.receive_ipsec_ok(principal_core.ccap_core_id)) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_REINIT_TCP) self.assertTrue( principal_core.agent_status[provision_pb2.AGENTTYPE_IPSEC]) self.assertTrue( self.recv_gcp_msg(principal_core.ccap_core_id, "TCP_FAIL")) if i == principal_core.CoreStateFailureRetry[ principal_core.fsm.STATE_REINIT_IPSEC]: break self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_REINIT_IPSEC) self.assertIsNotNone(principal_core.registered_timers[ provision_pb2.AGENTTYPE_IPSEC]) self.assertEquals( principal_core.state_retried_times[ principal_core.fsm.STATE_REINIT_IPSEC], i + 1) self.assertIsNotNone(principal_core.registered_timers[ provision_pb2.AGENTTYPE_IPSEC]) self.assertFalse( principal_core.action_status[provision_pb2.AGENTTYPE_IPSEC]) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_DEL) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_FAIL) self.assertIsNotNone(self.mgr.reboot_timer) def test_scenario_from_op_to_dhcp_fail(self): self.assertTrue(self.set_mgr_operational()) print "Principal tcp fail" principal_core = self.mgr.principal_core msg = TestCCAPCoreClass.build_event_notification( ccap_id=principal_core.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="TCP_FAIL", agentid=provision_pb2.AGENTTYPE_GCP) principal_core._handle_core_gcp_event(msg) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_REINIT_IPSEC) self.assertIsNotNone( principal_core.registered_timers[provision_pb2.AGENTTYPE_IPSEC]) self.assertEquals( principal_core.state_retried_times[ principal_core.fsm.STATE_REINIT_IPSEC], 1) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPAL_FOUND) self.assertFalse( principal_core.agent_status[provision_pb2.AGENTTYPE_IPSEC]) print "Principal ipsec up" msg = TestCCAPCoreClass.build_event_notification( ccap_id=principal_core.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="UP", agentid=provision_pb2.AGENTTYPE_IPSEC) principal_core._handle_core_ipsec_event(msg) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_REINIT_TCP) msg = TestCCAPCoreClass.build_event_notification( ccap_id=principal_core.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="TCP_FAIL", agentid=provision_pb2.AGENTTYPE_GCP) principal_core._handle_core_gcp_event(msg) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_REINIT_IPSEC) self.assertIsNotNone( principal_core.registered_timers[provision_pb2.AGENTTYPE_IPSEC]) self.assertEquals( principal_core.state_retried_times[ principal_core.fsm.STATE_REINIT_IPSEC], 2) self.assertFalse( principal_core.agent_status[provision_pb2.AGENTTYPE_IPSEC]) startupcore = self.get_startup_core() startupcore.CoreAgentTimeout[provision_pb2.AGENTTYPE_DHCP] = 1 print "startup core dhcp fail" startupcore.fsm.TRIGGER_DHCP_FAIL() self.assertIsNotNone( startupcore.registered_timers[provision_pb2.AGENTTYPE_DHCP]) self.assertEqual(startupcore.fsm.current, startupcore.fsm.STATE_8021X_OK) for state in principal_core.state_retried_times: self.assertEquals(principal_core.state_retried_times[state], 0) for agent_id in principal_core.agent_timeout: self.assertEquals(principal_core.agent_timeout[agent_id], 0) startupcore.agent_timeout[ provision_pb2.AGENTTYPE_DHCP] = startupcore.INIT_DHCP_MAX_TIMES - 1 print "Startup core dhcp fail reach max retries" startupcore.fsm.TRIGGER_DHCP_FAIL() self.assertEqual(startupcore.fsm.current, startupcore.fsm.STATE_8021X_OK) self.assertIsNotNone( startupcore.registered_timers[provision_pb2.AGENTTYPE_DHCP]) time.sleep(1) self.mgr.dispatcher.handle_one_event() print "system enter failure state" self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_FAIL) self.assertIsNotNone(self.mgr.reboot_timer) def test_scenario_from_op_to_recover_ok(self): self.assertTrue(self.set_mgr_operational()) startupcore = self.get_startup_core() print "startup core dhcp fail" startupcore.fsm.TRIGGER_DHCP_FAIL() principal_core = self.mgr.principal_core print "check principal hold at ipsec" self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_REINIT_IPSEC) for state in principal_core.state_retried_times: self.assertEquals(principal_core.state_retried_times[state], 0) for agent_id in principal_core.agent_timeout: self.assertEquals(principal_core.agent_timeout[agent_id], 0) self.assertFalse( principal_core.action_status[provision_pb2.AGENTTYPE_IPSEC]) startupcore.fsm.TRIGGER_DHCP_OK() startupcore.fsm.TRIGGER_TOD_OK() self.assertTrue( principal_core.action_status[provision_pb2.AGENTTYPE_IPSEC]) @staticmethod def build_mgr_event(mgrid, agentid, msg_str, data): event_request_rsp = process_agent_pb2.msg_event_notification() event_request_rsp.mgr_event.mgr_id = mgrid event_request_rsp.mgr_event.event_id = agentid event_request_rsp.mgr_event.data = json.dumps(msg_str + "/" + json.dumps(data)) return event_request_rsp.mgr_event @staticmethod def get_core(interface, ip): for ccap_core in CCAPCore.ccap_core_db.values(): if ccap_core.interface == interface and ccap_core.ccap_core_network_address == ip: return ccap_core return None def test_scenario_from_op_to_add_lcha_core(self): self.assertTrue(self.set_mgr_operational()) print "Test 3.3.3.3 is not in DB, ignore this message" caps = { "index": 3, "is_active": False, "ccap_core": "3.3.3.3", "interface": 'eth0', "is_principal": True } msg = self.build_mgr_event(mgrid=self.mgr.mgr_id, agentid=provision_pb2.AGENTTYPE_GCP, msg_str='role', data=caps) try: self.mgr._handle_mgr_event_notification( msg=msg, agent_id=provision_pb2.AGENTTYPE_GCP) except Exception as e: self.fail("Exception happened : %s" % e) print "start try 2.2.2.2, set it to aux active" self.assertEquals(len(CCAPCore.ccap_core_db), 3) caps = { "index": 2, "is_active": True, "ccap_core": "2.2.2.2", "interface": 'eth0', "is_principal": False } msg = self.build_mgr_event(mgrid=self.mgr.mgr_id, agentid=provision_pb2.AGENTTYPE_GCP, msg_str='role', data=caps) try: self.mgr._handle_mgr_event_notification( msg=msg, agent_id=provision_pb2.AGENTTYPE_GCP) except Exception as e: self.fail("Exception happened : %s" % e) aux_active = self.get_core("eth0", '2.2.2.2') self.assertIsNotNone(aux_active) self.assertIsInstance(aux_active, CCAPCore) self.assertTrue( self.set_core_ready_for_tcp(coreid=aux_active.ccap_core_id)) self.assertTrue(self.set_core_op(aux_active.ccap_core_id)) print "Add 3.3.3.3 as standby core" ha_info = { "ActiveCoreIpAddress": "2.2.2.2", "StandbyCoreIpAddress": "3.3.3.3", "interface": "eth0", "operation": self.mgr.OPERATION_ADD } msg = self.build_mgr_event(mgrid=self.mgr.mgr_id, agentid=provision_pb2.AGENTTYPE_GCP, msg_str='Ha', data=ha_info) self.mgr._handle_mgr_event_notification( msg=msg, agent_id=provision_pb2.AGENTTYPE_GCP) self.assertEquals(len(CCAPCore.ccap_core_db), 4) caps = { "index": 3, "is_active": False, "ccap_core": "3.3.3.3", "interface": 'eth0', "is_principal": True } msg = self.build_mgr_event(mgrid=self.mgr.mgr_id, agentid=provision_pb2.AGENTTYPE_GCP, msg_str='role', data=caps) try: self.mgr._handle_mgr_event_notification( msg=msg, agent_id=provision_pb2.AGENTTYPE_GCP) except Exception as e: self.fail("Exception happened : %s" % e) aux_standby = self.get_core("eth0", '3.3.3.3') self.assertIsNotNone(aux_standby) self.assertIsInstance(aux_standby, CCAPCore) aux_standby.fsm.TRIGGER_Startup() self.assertTrue( self.set_core_ready_for_tcp(coreid=aux_standby.ccap_core_id)) self.assertTrue(self.set_core_op(aux_standby.ccap_core_id)) def test_lcha_change_failed(self): # this case is operations based on core added self.test_scenario_from_op_to_add_lcha_core() print "Wrong change information, nothing changed" ha_info = { "ActiveCoreIpAddress": "3.3.3.3", "StandbyCoreIpAddress": "2.2.2.2", "interface": "eth0", "operation": self.mgr.OPERATION_CHANGE } msg = self.build_mgr_event(mgrid=self.mgr.mgr_id, agentid=provision_pb2.AGENTTYPE_GCP, msg_str='Ha', data=ha_info) self.mgr._handle_mgr_event_notification( msg=msg, agent_id=provision_pb2.AGENTTYPE_GCP) self.assertIsNotNone(self.get_core('eth0', '2.2.2.2')) self.assertIsNotNone(self.get_core('eth0', '3.3.3.3')) def test_lcha_change_success(self): # this case is operations based on core added self.test_scenario_from_op_to_add_lcha_core() print "Receive correct lcha change information, do the change." ha_info = { "ActiveCoreIpAddress": "2.2.2.2", "StandbyCoreIpAddress": "3.3.3.3", "interface": "eth0", "operation": self.mgr.OPERATION_CHANGE } msg = self.build_mgr_event(mgrid=self.mgr.mgr_id, agentid=provision_pb2.AGENTTYPE_GCP, msg_str='Ha', data=ha_info) self.mgr._handle_mgr_event_notification( msg=msg, agent_id=provision_pb2.AGENTTYPE_GCP) self.assertIsNone(self.get_core('eth0', '2.2.2.2')) self.assertIsNotNone(self.get_core('eth0', '3.3.3.3')) self.mgr.core_orchestrator.orchestrator_cb(None) self.mgr.core_orchestrator.orchestrator_cb(None) self.assertIsNotNone(self.get_core('eth0', '2.2.2.2')) def test_remove_principal_from_operational(self): self.assertTrue(self.set_mgr_operational()) print "Manager receive core identification" caps = { "index": 1, "is_active": True, "ccap_core": self.mgr.principal_core.ccap_core_network_address, 'interface': self.mgr.principal_core.interface, "is_principal": False } event_request_rsp = process_agent_pb2.msg_event_notification() event_request_rsp.mgr_event.mgr_id = self.mgr.mgr_id event_request_rsp.mgr_event.event_id = provision_pb2.AGENTTYPE_GCP event_request_rsp.mgr_event.data = json.dumps("role/" + json.dumps(caps)) self.mgr._handle_mgr_gcp_event(event_request_rsp.mgr_event) self.assertIsNone(self.mgr.principal_core) def test_scenario_startup_exit_online(self): # test startup core exit online state, with gcp cores in [init(ipsec), online] self.assertTrue(self.set_mgr_operational()) gcpcore = self.get_core("eth0", '2.2.2.2') self.assertEqual(gcpcore.fsm.current, gcpcore.fsm.STATE_INIT_IPSEC) # startup exit online because of dhcp failed startupcore = self.get_startup_core() print "Manager DHCP OK" msg = TestCCAPCoreClass.build_event_notification( ccap_id=startupcore.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="DOWN", agentid=provision_pb2.AGENTTYPE_DHCP) startupcore._handle_core_dhcp_event(msg) self.assertEqual(startupcore.fsm.current, CCAPFsmStartup.STATE_8021X_OK) self.assertEqual(self.mgr.principal_core.fsm.current, CCAPFsm.STATE_REINIT_IPSEC) self.assertEqual(gcpcore.fsm.current, CCAPFsm.STATE_INIT_IPSEC) # startup core receive an INTERFACE DOWN msg = TestCCAPCoreClass.build_event_notification( ccap_id=startupcore.ccap_core_id, status=process_agent_pb2.msg_core_event_notification.OK, reason="test", result="DOWN", agentid=provision_pb2.AGENTTYPE_INTERFACE_STATUS) startupcore._handle_core_interface_event(msg) self.assertEquals(startupcore.fsm.current, CCAPFsmStartup.STATE_INIT) self.assertEqual(self.mgr.principal_core.fsm.current, CCAPFsm.STATE_REINIT_IPSEC) self.assertEqual(gcpcore.fsm.current, CCAPFsm.STATE_INIT_IPSEC) def test_scenario_gcp_close(self): self.assertTrue(self.set_mgr_principal_found()) coreid = self.mgr.principal_core.ccap_core_id principal_core = self.mgr.principal_core self.assertTrue(self.recv_gcp_msg(coreid, "TCP_OK")) self.assertEqual(principal_core.fsm.current, CCAPFsm.STATE_INIT_GCP_IRA) self.assertTrue(self.recv_gcp_msg(coreid, "GCP_IRA")) self.assertEqual(principal_core.fsm.current, CCAPFsm.STATE_INIT_GCP_CFG) self.assertTrue(self.recv_gcp_msg(coreid, "GCP_CFG")) self.assertEqual(principal_core.fsm.current, CCAPFsm.STATE_INIT_GCP_CFG_CPL) self.assertEqual( principal_core.state_retried_times[ principal_core.fsm.STATE_INIT_IPSEC], 0) self.assertTrue( self.recv_gcp_msg(principal_core.ccap_core_id, "TCP_FAIL")) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_INIT_IPSEC) self.assertIsNotNone( principal_core.registered_timers[provision_pb2.AGENTTYPE_IPSEC]) self.assertEquals( principal_core.state_retried_times[ principal_core.fsm.STATE_INIT_IPSEC], 1) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_PRINCIPAL_FOUND) self.assertFalse( principal_core.agent_status[provision_pb2.AGENTTYPE_IPSEC]) self.assertTrue( principal_core.action_status[provision_pb2.AGENTTYPE_IPSEC]) for i in range( 1, principal_core.CoreStateFailureRetry[ principal_core.fsm.STATE_INIT_IPSEC] + 1): self.assertTrue(self.receive_ipsec_ok(principal_core.ccap_core_id)) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_INIT_TCP) self.assertTrue( principal_core.agent_status[provision_pb2.AGENTTYPE_IPSEC]) self.assertTrue(self.recv_gcp_msg(coreid, "TCP_OK")) self.assertEqual(principal_core.fsm.current, CCAPFsm.STATE_INIT_GCP_IRA) self.assertTrue(self.recv_gcp_msg(coreid, "GCP_IRA")) self.assertEqual(principal_core.fsm.current, CCAPFsm.STATE_INIT_GCP_CFG) self.assertTrue(self.recv_gcp_msg(coreid, "GCP_CFG")) self.assertEqual(principal_core.fsm.current, CCAPFsm.STATE_INIT_GCP_CFG_CPL) self.assertTrue( self.recv_gcp_msg(principal_core.ccap_core_id, "TCP_FAIL")) if i == principal_core.CoreStateFailureRetry[ principal_core.fsm.STATE_INIT_IPSEC]: break self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_INIT_IPSEC) self.assertIsNotNone(principal_core.registered_timers[ provision_pb2.AGENTTYPE_IPSEC]) self.assertEquals( principal_core.state_retried_times[ principal_core.fsm.STATE_INIT_IPSEC], i + 1) self.assertIsNotNone(principal_core.registered_timers[ provision_pb2.AGENTTYPE_IPSEC]) self.assertFalse( principal_core.action_status[provision_pb2.AGENTTYPE_IPSEC]) self.assertEquals(principal_core.fsm.current, principal_core.fsm.STATE_DEL) self.assertEquals(self.mgr.fsm.current, self.mgr.fsm.STATE_FAIL) self.assertIsNotNone(self.mgr.reboot_timer)