def test_sys_tools(self): SysTools.touch('/tmp/test_utils') SysTools.REBOOT_SKIP_FILES = ('/tmp/test_utils') self.assertFalse(SysTools.is_system_openwrt()) self.assertEqual(SysTools.get_mac_address('if_test'), "00:00:00:00:00:00") self.assertEqual(SysTools.get_sys_mac_address(), "00:00:00:00:00:00") with open('/tmp/test_mac', 'w') as f: f.write('01:02:03:04:05:06') f.close() SysTools.SYS_MAC_FILE = '/tmp/test_mac' self.assertEqual(SysTools.get_sys_mac_address(), "01:02:03:04:05:06") SysTools.SYS_MAC_FILE = '/etc/gshadow' self.assertEqual(SysTools.get_sys_mac_address(), "00:00:00:00:00:00") print(SysTools.get_host_name()) print(SysTools.get_ip_address('if_test')) if not SysTools.reboot(): SysTools.sys_failure_reboot('skip files exist') SysTools.external_reboot(('test', 'test')) if os.path.exists('/tmp/test_utils'): os.remove('/tmp/test_utils') # d = Dispatcher() # timestamp = time.time() + 1000 # SysTools.set_system_time(d, timestamp) # SysTools.touch('/etc/openwrt_release') # SysTools.set_system_time(d, timestamp) # time.sleep(2) # self.assertTrue(timestamp > time.time()) #should no permission to change system time # del d # if os.path.exists('/etc/openwrt_release'): # os.remove('/etc/openwrt_release') self.assertIsNotNone(SysTools.sys_up_time()) ret = SysTools.if_indextoname(1) print "ifname of index 1:", ret self.assertIsNotNone(ret) ret = SysTools.if_indextoname("test") self.assertIsNone(ret) ret = SysTools.if_indextoname(600) self.assertIsNone(ret) ret = SysTools.if_nametoindex("lo") print "ifindex of lo:", ret self.assertIsNotNone(ret) ret = SysTools.if_nametoindex(5) self.assertIsNone(ret) ret = SysTools.if_nametoindex("dafsd") self.assertIsNone(ret) ret = SysTools.is_if_oper_up('eth0') self.assertTrue(ret)
def start(self): # pragma: no cover """Start all agent here, need to pay attention to the error case when start agent fail.""" # Start the HAL and Fake Driver if self.hal_cmd_line: self.logger.info("Start the hal main process...") process_obj = self.start_process(self.hal_cmd_line) self.hal_process = { "process": process_obj, "retries": 0, } manager_debugability.debugability_process_monitor(self.hal_process) # wait a period for process start and init complete time.sleep(self.PROCESS_INIT_PERIOD) if self.fake_driver_cmd_line: self.logger.info("Start the fake driver process...") process_obj = self.start_process(self.fake_driver_cmd_line) self.fake_driver_process = { "process": process_obj, "retries": 0, } manager_debugability.debugability_process_monitor( self.fake_driver_process) for agent_name in self.agent_dict: self.logger.info("start agent process {}...".format(agent_name)) popenObj = self.start_process(self.agent_dict[agent_name]) self.agent_obj[agent_name] = { "process": popenObj, "retries": 0, } manager_debugability.debugability_process_monitor( self.agent_obj[agent_name]) # wait a period for agent start and init complete alive_status = False for timeout in range(self.WAITING_FOR_AGENT_STARTUP_RETRY): alive_status = ProcessAgent.is_all_agent_started() if not alive_status: time.sleep(1) if not alive_status: self.logger.error( 'Not all agent startup normally, reboot the system.') SysTools.sys_failure_reboot(reason='Not all agent startup') SysTools.diagnostic_self_test_fail('Communication error', 'Not all agent startup', 'Severity level=error') # start the manager process self.logger.info("Start the manager process...") process_obj = self.start_process(self.manager_cmd_line) self.manager_process = { "process": process_obj, "retries": 0, } manager_debugability.debugability_process_monitor(self.manager_process) # start the fault_manager process self.logger.info("Start the fault manager process...") process_obj = self.start_process(self.fault_manager_cmd_line) self.manager_process = { "process": process_obj, "retries": 0, } manager_debugability.debugability_process_monitor(self.manager_process) if self.ptp_driver_cmd_line: self.logger.info("Start the ptp driver client process...") process_obj = self.start_process(self.ptp_driver_cmd_line) self.ptp_driver_process = { "process": process_obj, "retries": 0, } manager_debugability.debugability_process_monitor( self.ptp_driver_process) if self.res_hal_cmd_line: self.logger.info("Start the resource hal client process...") process_obj = self.start_process(self.res_hal_cmd_line) self.res_driver_process = { "process": process_obj, "retries": 0, } manager_debugability.debugability_process_monitor( self.res_driver_process) if self.ssd_driver_cmd_line: self.logger.info("Start the ssd driver client process...") process_obj = self.start_process(self.ssd_driver_cmd_line) self.ssd_driver_process = { "process": process_obj, "retries": 0, } manager_debugability.debugability_process_monitor( self.ssd_driver_process) while True: time.sleep(5) # monitor the all process manager_debugability.debugability_traceback() # monitor the manager process, will not retry.... if self.manager_process is not None and self.manager_process[ 'process'] is None: self.logger.error( "Manager process is not up, reboot the system.") if self.simulator_flag: sys.exit(-1) else: SysTools.sys_failure_reboot( reason="Manager process is not up") SysTools.diagnostic_self_test_fail( 'Processing error', 'Manager process is not up', 'Severity level=error') for agent in self.agent_obj: # check if agent instance create succeed, retry if failure if None is self.agent_obj[agent]["process"]: if self.agent_obj[agent][ "retries"] < self.AGENT_RETRIES_MAX: self.logger.warn('Agent %s retries %d times', agent, self.agent_obj[agent]["retries"]) self.agent_obj[agent]["process"] = self.start_process( self.agent_dict[agent_name]) self.agent_obj[agent]["retries"] += 1 self.logger.warn('Agent %s retries %d times', agent, self.agent_obj[agent]["retries"]) manager_debugability.debugability_process_monitor( self.agent_obj[agent]) continue else: # FixMe: reboot system or ? self.logger.error( 'Agent %s retries times exceed, will reboot...', agent) SysTools.sys_failure_reboot( reason="Agent {0} retries times exceed".format( agent)) SysTools.diagnostic_self_test_fail( 'Communication error', "Agent {0} retries times exceed".format(agent), 'Severity level=error') if self.check_process_status(self.agent_obj[agent]["process"] ) != self.PROCESSSTATE_ALIVE: self.logger.error('%s process is dead, reboot the system.', agent) # FixMe: reboot system or restart agent SysTools.sys_failure_reboot( reason="{0} process is dead".format(agent)) SysTools.diagnostic_self_test_fail( 'Processing error', "{0} process is dead".format(agent), 'Severity level=error') # check other critical processes if self.ptp_driver_cmd_line: if self.check_process_status(self.ptp_driver_process["process"] ) != self.PROCESSSTATE_ALIVE: self.logger.error("ptp hal driver process is dead") SysTools.sys_failure_reboot( reason="ptp hal driver process is dead") SysTools.diagnostic_self_test_fail( 'Processing error', "ptp hal driver process is dead", 'Severity level=error')