def _execute_test(cls, test: FirmwareTestClass, router: Router, routers: List[Router]) -> TestResult:
if not isinstance(router, Router):
raise ValueError("Chosen Router is not a real Router...")
# proofed: this method runs in other process as the server
setproctitle(str(router.id) + " - " + str(test))
logging.debug("%sExecute test " + str(test) + " on Router(" + str(router.id) + ")", LoggerSetup.get_log_deep(2))
test_suite = defaultTestLoader.loadTestsFromTestCase(test)
# prepare all test cases
for test_case in test_suite:
logging.debug("%sTestCase " + str(test_case), LoggerSetup.get_log_deep(4))
test_case.prepare(router, routers)
result = TestResult()
cls.__setns(router)
try:
result = test_suite.run(result)
except Exception as e:
logging.error("%sTestCase execution raised an exception", LoggerSetup.get_log_deep(3))
logging.error("%s" + str(e), LoggerSetup.get_log_deep(3))
test_obj = test()
result.addError(test_obj, sys.exc_info()) # add the reason of the exception
finally:
# I'm sry for this dirty hack, but if you don't do this you get an
# "TypeError: cannot serialize '_io.TextIOWrapper' object" because sys.stdout is not serializeable...
result._original_stdout = None
result._original_stderr = None
logging.debug("%sResult from test " + str(result), LoggerSetup.get_log_deep(3))
return result
def _wait_for_test_done(cls, test: FirmwareTestClass, router: Router, done_event: DoneEvent) -> None:
"""
Wait 2 minutes until the test is done.
Handles the result from the tests.
Triggers the next job/test.
:param test: test to execute
:param router: the Router
:param done_event: event which will be triggered when the task is finished
"""
logging.debug("%sWait for test" + str(test), LoggerSetup.get_log_deep(2))
try:
async_result = cls._task_pool.apply_async(func=cls._execute_test, args=(test, router, cls._routers))
result = async_result.get(120) # wait 2 minutes or raise an TimeoutError
logging.debug("%sTest done " + str(test), LoggerSetup.get_log_deep(1))
logging.debug("%sFrom Router(" + str(router.id) + ")", LoggerSetup.get_log_deep(2))
cls._test_results.append((router.id, str(test), result))
try:
length = len(cls._test_results)
t = cls._test_results[(length - 1)]
cls.write_in_db(str(length), t)
except Exception as e:
logging.error("Error at write test results into DB: {0}".format(e))
except Exception as e:
logging.error("%sTest raised an Exception: " + str(e), LoggerSetup.get_log_deep(1))
result = TestResult()
result._original_stdout = None
result._original_stderr = None
cls._test_results.append((router.id, str(test), result))
cls._task_errors.append((router.id, sys.exc_info()))
try:
length = len(cls._test_results)
t = cls._test_results[(length - 1)]
cls.write_in_db(str(length), t)
except Exception as e:
logging.error("Error at write test results into DB: {0}".format(e))
finally:
cls.set_running_task(router, None)
# logging.debug(str(cls._test_results))
# start next test in the queue
done_event.set()
cls.__start_task(router, None)
def _wait_for_test_done(cls, test: FirmwareTestClass, router: Router,
done_event: DoneEvent) -> None:
"""
Wait 5 minutes until the test is done.
Handles the result from the tests.
Triggers the next job/test.
:param test: test to execute
:param router: the Router
"""
logging.debug("%sWait for test" + str(test),
LoggerSetup.get_log_deep(2))
try:
async_result = cls._task_pool.apply_async(func=cls._execute_test,
args=(test, router))
result = async_result.get(
300) # wait 5 minutes or raise an TimeoutError
logging.debug("%sTest done " + str(test),
LoggerSetup.get_log_deep(1))
logging.debug("%sFrom " + str(router), LoggerSetup.get_log_deep(2))
cls._test_results.append((router.id, str(test), result))
except Exception as e:
# TODO #105
logging.error("%sTest raised an Exception: " + str(e),
LoggerSetup.get_log_deep(1))
result = TestResult()
result._original_stdout = None
result._original_stderr = None
# result.addError(None, (type(exception), exception, None))
# TODO exception handling for failed Tests
cls._test_results.append((router.id, str(test), result))
finally:
cls.set_running_task(router, None)
# logging.debug(str(cls._test_results))
# start next test in the queue
done_event.set()
cls.__start_task(router, None)
def _execute_test(cls, test: FirmwareTestClass,
router: Router) -> TestResult:
if not isinstance(router, Router):
raise ValueError("Chosen Router is not a real Router...")
# proofed: this method runs in other process as the server
logging.debug("%sExecute test " + str(test) + " on " + str(router),
LoggerSetup.get_log_deep(2))
test_suite = defaultTestLoader.loadTestsFromTestCase(test)
# prepare all test cases
for test_case in test_suite:
logging.debug("%sTestCase " + str(test_case),
LoggerSetup.get_log_deep(4))
test_case.prepare(router)
result = TestResult()
cls.__setns(router)
try:
result = test_suite.run(
result) # TODO if debug set, run as debug()
except Exception as e:
logging.error("%sTestCase raised an exception",
LoggerSetup.get_log_deep(3))
logging.error("%s" + str(e), LoggerSetup.get_log_deep(3))
finally:
# I'm sry for this dirty hack, but if you don't do this you get an
# "TypeError: cannot serialize '_io.TextIOWrapper' object" because sys.stdout is not serializeable...
result._original_stdout = None
result._original_stderr = None
logging.debug("%sResult from test " + str(result),
LoggerSetup.get_log_deep(3))
return result
def start(cls, config_path: str = CONFIG_PATH) -> None:
"""
Starts the runtime server with all components.
:param config_path: Path to an alternative config directory
"""
# server has to be run with root rights - except on travis CI
if not os.geteuid() == 0 and not os.environ.get('TRAVIS'):
sys.exit('Script must be run as root')
cls._stopped = Lock()
signal.signal(signal.SIGTERM, cls._signal_term_handler)
cls.CONFIG_PATH = config_path
# set the config_path at the manager
ConfigManager.set_config_path(config_path)
# read from config the Vlan mode
vlan_activate = ConfigManager.get_server_property("Vlan_On")
cls.VLAN = vlan_activate
# read from config if debug mode is on
log_level = int(ConfigManager.get_server_property("Log_Level"))
debug_mode = False
if log_level is 10:
debug_mode = True
cls.DEBUG = debug_mode
setproctitle("fftserver")
cls._server_stop_event = Event()
cls._pid = os.getpid()
# create instance and give params to the logger object
LoggerSetup.setup(log_level)
# load Router configs
cls.__load_configuration()
for router in cls.get_routers():
cls._running_task.append(None)
cls._waiting_tasks.append(deque())
# start process/thread pool for job and test handling
cls._max_subprocesses = (len(cls._routers) + 1) # plus one for the power strip
cls._task_pool = Pool(processes=cls._max_subprocesses, initializer=init_process,
initargs=(cls._server_stop_event,), maxtasksperchild=1)
cls._task_wait_executor = ThreadPoolExecutor(max_workers=(cls._max_subprocesses * 2))
# start thread for multiprocess stop wait
t = threading.Thread(target=cls._close_wait)
t.start()
# add Namespace and Vlan for each Router
if cls.VLAN:
cls._nv_assistent = NVAssistent("eth0")
for router in cls.get_routers():
logging.debug("Add Namespace and Vlan for Router(" + str(router.id) + ")")
cls._nv_assistent.create_namespace_vlan(router)
# add Namespace and Vlan for 1 Powerstrip (expand to more if necessary)
logging.debug("Add Namespace and Vlan for Powerstrip")
cls._nv_assistent.create_namespace_vlan(cls.get_power_strip())
# update Router
cls.router_online(None, update_all=True, blocked=True)
cls.update_router_info(None, update_all=True)
# open database and read old test results
try:
with shelve.open('test_results', 'c') as db:
# read test values
key_list = db.keys()
for k in key_list:
t = TestResult()
dbt = db[str(k)]
t.failures = dbt.failures
t.errors = dbt.errors
t.testsRun = dbt.testsRun
t._original_stdout = None
t._original_stderr = None
cls._test_results.append((dbt.router_id, dbt.test_name, t))
except Exception as e:
logging.error("Error at read test results from DB: {0}".format(e))
logging.info("Runtime Server started")
try:
cls._ipc_server.start_ipc_server(cls, True) # serves forever - works like a while(true)
except (KeyboardInterrupt, SystemExit):
logging.info("Received an interrupt signal")
cls.stop()
