def _execute_test(cls, test: FirmwareTestClass, router: Router, routers: List[Router]) -> TestResult: if not isinstance(router, Router): raise ValueError("Chosen Router is not a real Router...") # proofed: this method runs in other process as the server setproctitle(str(router.id) + " - " + str(test)) logging.debug("%sExecute test " + str(test) + " on Router(" + str(router.id) + ")", LoggerSetup.get_log_deep(2)) test_suite = defaultTestLoader.loadTestsFromTestCase(test) # prepare all test cases for test_case in test_suite: logging.debug("%sTestCase " + str(test_case), LoggerSetup.get_log_deep(4)) test_case.prepare(router, routers) result = TestResult() cls.__setns(router) try: result = test_suite.run(result) except Exception as e: logging.error("%sTestCase execution raised an exception", LoggerSetup.get_log_deep(3)) logging.error("%s" + str(e), LoggerSetup.get_log_deep(3)) test_obj = test() result.addError(test_obj, sys.exc_info()) # add the reason of the exception finally: # I'm sry for this dirty hack, but if you don't do this you get an # "TypeError: cannot serialize '_io.TextIOWrapper' object" because sys.stdout is not serializeable... result._original_stdout = None result._original_stderr = None logging.debug("%sResult from test " + str(result), LoggerSetup.get_log_deep(3)) return result
def _wait_for_test_done(cls, test: FirmwareTestClass, router: Router, done_event: DoneEvent) -> None: """ Wait 2 minutes until the test is done. Handles the result from the tests. Triggers the next job/test. :param test: test to execute :param router: the Router :param done_event: event which will be triggered when the task is finished """ logging.debug("%sWait for test" + str(test), LoggerSetup.get_log_deep(2)) try: async_result = cls._task_pool.apply_async(func=cls._execute_test, args=(test, router, cls._routers)) result = async_result.get(120) # wait 2 minutes or raise an TimeoutError logging.debug("%sTest done " + str(test), LoggerSetup.get_log_deep(1)) logging.debug("%sFrom Router(" + str(router.id) + ")", LoggerSetup.get_log_deep(2)) cls._test_results.append((router.id, str(test), result)) try: length = len(cls._test_results) t = cls._test_results[(length - 1)] cls.write_in_db(str(length), t) except Exception as e: logging.error("Error at write test results into DB: {0}".format(e)) except Exception as e: logging.error("%sTest raised an Exception: " + str(e), LoggerSetup.get_log_deep(1)) result = TestResult() result._original_stdout = None result._original_stderr = None cls._test_results.append((router.id, str(test), result)) cls._task_errors.append((router.id, sys.exc_info())) try: length = len(cls._test_results) t = cls._test_results[(length - 1)] cls.write_in_db(str(length), t) except Exception as e: logging.error("Error at write test results into DB: {0}".format(e)) finally: cls.set_running_task(router, None) # logging.debug(str(cls._test_results)) # start next test in the queue done_event.set() cls.__start_task(router, None)
def _wait_for_test_done(cls, test: FirmwareTestClass, router: Router, done_event: DoneEvent) -> None: """ Wait 5 minutes until the test is done. Handles the result from the tests. Triggers the next job/test. :param test: test to execute :param router: the Router """ logging.debug("%sWait for test" + str(test), LoggerSetup.get_log_deep(2)) try: async_result = cls._task_pool.apply_async(func=cls._execute_test, args=(test, router)) result = async_result.get( 300) # wait 5 minutes or raise an TimeoutError logging.debug("%sTest done " + str(test), LoggerSetup.get_log_deep(1)) logging.debug("%sFrom " + str(router), LoggerSetup.get_log_deep(2)) cls._test_results.append((router.id, str(test), result)) except Exception as e: # TODO #105 logging.error("%sTest raised an Exception: " + str(e), LoggerSetup.get_log_deep(1)) result = TestResult() result._original_stdout = None result._original_stderr = None # result.addError(None, (type(exception), exception, None)) # TODO exception handling for failed Tests cls._test_results.append((router.id, str(test), result)) finally: cls.set_running_task(router, None) # logging.debug(str(cls._test_results)) # start next test in the queue done_event.set() cls.__start_task(router, None)
def _execute_test(cls, test: FirmwareTestClass, router: Router) -> TestResult: if not isinstance(router, Router): raise ValueError("Chosen Router is not a real Router...") # proofed: this method runs in other process as the server logging.debug("%sExecute test " + str(test) + " on " + str(router), LoggerSetup.get_log_deep(2)) test_suite = defaultTestLoader.loadTestsFromTestCase(test) # prepare all test cases for test_case in test_suite: logging.debug("%sTestCase " + str(test_case), LoggerSetup.get_log_deep(4)) test_case.prepare(router) result = TestResult() cls.__setns(router) try: result = test_suite.run( result) # TODO if debug set, run as debug() except Exception as e: logging.error("%sTestCase raised an exception", LoggerSetup.get_log_deep(3)) logging.error("%s" + str(e), LoggerSetup.get_log_deep(3)) finally: # I'm sry for this dirty hack, but if you don't do this you get an # "TypeError: cannot serialize '_io.TextIOWrapper' object" because sys.stdout is not serializeable... result._original_stdout = None result._original_stderr = None logging.debug("%sResult from test " + str(result), LoggerSetup.get_log_deep(3)) return result
def start(cls, config_path: str = CONFIG_PATH) -> None: """ Starts the runtime server with all components. :param config_path: Path to an alternative config directory """ # server has to be run with root rights - except on travis CI if not os.geteuid() == 0 and not os.environ.get('TRAVIS'): sys.exit('Script must be run as root') cls._stopped = Lock() signal.signal(signal.SIGTERM, cls._signal_term_handler) cls.CONFIG_PATH = config_path # set the config_path at the manager ConfigManager.set_config_path(config_path) # read from config the Vlan mode vlan_activate = ConfigManager.get_server_property("Vlan_On") cls.VLAN = vlan_activate # read from config if debug mode is on log_level = int(ConfigManager.get_server_property("Log_Level")) debug_mode = False if log_level is 10: debug_mode = True cls.DEBUG = debug_mode setproctitle("fftserver") cls._server_stop_event = Event() cls._pid = os.getpid() # create instance and give params to the logger object LoggerSetup.setup(log_level) # load Router configs cls.__load_configuration() for router in cls.get_routers(): cls._running_task.append(None) cls._waiting_tasks.append(deque()) # start process/thread pool for job and test handling cls._max_subprocesses = (len(cls._routers) + 1) # plus one for the power strip cls._task_pool = Pool(processes=cls._max_subprocesses, initializer=init_process, initargs=(cls._server_stop_event,), maxtasksperchild=1) cls._task_wait_executor = ThreadPoolExecutor(max_workers=(cls._max_subprocesses * 2)) # start thread for multiprocess stop wait t = threading.Thread(target=cls._close_wait) t.start() # add Namespace and Vlan for each Router if cls.VLAN: cls._nv_assistent = NVAssistent("eth0") for router in cls.get_routers(): logging.debug("Add Namespace and Vlan for Router(" + str(router.id) + ")") cls._nv_assistent.create_namespace_vlan(router) # add Namespace and Vlan for 1 Powerstrip (expand to more if necessary) logging.debug("Add Namespace and Vlan for Powerstrip") cls._nv_assistent.create_namespace_vlan(cls.get_power_strip()) # update Router cls.router_online(None, update_all=True, blocked=True) cls.update_router_info(None, update_all=True) # open database and read old test results try: with shelve.open('test_results', 'c') as db: # read test values key_list = db.keys() for k in key_list: t = TestResult() dbt = db[str(k)] t.failures = dbt.failures t.errors = dbt.errors t.testsRun = dbt.testsRun t._original_stdout = None t._original_stderr = None cls._test_results.append((dbt.router_id, dbt.test_name, t)) except Exception as e: logging.error("Error at read test results from DB: {0}".format(e)) logging.info("Runtime Server started") try: cls._ipc_server.start_ipc_server(cls, True) # serves forever - works like a while(true) except (KeyboardInterrupt, SystemExit): logging.info("Received an interrupt signal") cls.stop()