def reconnect_bus(self):
'''
need reconnect bus by self.bus's infos
:return:
'''
self.bus_client = BusClient(self.bus_client.domain,
self.bus_client.port,
self.bus_client.auth_key)
def msg_write(self, msg):
'''
write to local or not
:param msg:
:return:
'''
if self.bus_client is None:
self.bus_client = BusClient()
if self.local_logger:
self.write_local_logger(msg)
else:
if msg.get('level') == 'debug' and not self.debug_:
return
self.log.put(msg)
def web_server(bus_client: BusClient):
bus_client = BusClient(bus_client.domain, bus_client.port, bus_client.auth_key)
api.add_resource(LoaderResource, "/loader")
api.add_resource(RunnerResource, "/runner")
api.add_resource(ResultResource, "/result")
api.add_resource(DocResource, "/doc")
app.run(debug=False, port=WEB_SERVER_PORT)
class LoaderResource(Resource):
def __init__(self):
super().__init__()
self.bus_client = BusClient()
def get(self):
get_queue = self.bus_client.get_publish_loader()
if not get_queue.empty():
loader = get_queue.get()
globalenv.set_global("loader", loader)
return globalenv.get_global("loader")
class RunnerResource(Resource):
def __init__(self):
super().__init__()
self.bus_client = BusClient()
def get(self):
get_queue = self.bus_client.get_publish_runner()
runners = globalenv.get_global("runners")
if not get_queue.empty():
runner = get_queue.get()
runners[runner.get("key")] = runner
globalenv.set_global("runners", runners)
return globalenv.get_global("runners")
class ResultResource(Resource):
def __init__(self):
super().__init__()
self.bus_client = BusClient()
def get(self):
get_queue = self.bus_client.get_publish_result()
results = globalenv.get_global("results")
if not get_queue.empty():
results = get_queue.get()
globalenv.set_global("results", results)
return results.deserialize()
def _bus_client(self, args):
'''
init bus client
:param args:
:return:
'''
if isinstance(args.bus_server, list):
self.bus_client = BusClient(args.bus_server[1], args.bus_server[2], args.bus_server[0])
elif args.bus_server_port:
self.bus_client = BusClient(None, args.bus_server_port, None)
else:
self.bus_client = BusClient()
self.params_loader = self.bus_client.get_param()
def bind_busclient(self, bus_client: BusClient):
self.bus_client = BusClient(bus_client.domain, bus_client.port,
bus_client.auth_key)
self.log = self.bus_client.get_log()
class BaseLogger(metaclass=SingletonType):
'''
BaseLogger
SingletonType class
'''
def __init__(self, logger_name):
self.logger_name = logger_name
self.bus_client = None
self.process_id = 0
# here using this function to reconnect to the bus : loader, runner, recorder
def bind_busclient(self, bus_client: BusClient):
self.bus_client = BusClient(bus_client.domain, bus_client.port,
bus_client.auth_key)
self.log = self.bus_client.get_log()
# bind the pid of the process
def bind_process(self, process_id):
self.process_id = process_id
# set out put or not, or using local logger
def set_output(self, local_logger, nout=True, debug=False):
self.local_logger = local_logger
self.nout = nout
self.debug_ = debug
def debug(self, msg, logger_name=None):
msg = {
"process": self.process_id,
"logger_name":
self.logger_name if not logger_name else logger_name,
"level": "debug",
"msg": msg
}
self.msg_write(msg)
def info(self, msg, logger_name=None):
msg = {
"process": self.process_id,
"logger_name":
self.logger_name if not logger_name else logger_name,
"level": "info",
"msg": msg
}
self.msg_write(msg)
def warning(self, msg, logger_name=None):
msg = {
"process": self.process_id,
"logger_name":
self.logger_name if not logger_name else logger_name,
"level": "warning",
"msg": msg
}
self.msg_write(msg)
def error(self, msg, logger_name=None):
msg = {
"process": self.process_id,
"logger_name":
self.logger_name if not logger_name else logger_name,
"level": "error",
"msg": msg
}
self.msg_write(msg)
def critical(self, msg, logger_name=None):
msg = {
"process": self.process_id,
"logger_name":
self.logger_name if not logger_name else logger_name,
"level": "critical",
"msg": msg
}
self.msg_write(msg)
def __new__(cls, *args, **kwargs):
return object.__new__(cls)
def log_output(self, msg, origin_msg):
'''
print msg with params nout
:param msg:
:return: None
'''
if not self.nout:
if origin_msg.get('level') == 'debug' and not self.debug_:
return
print(msg)
def write_local_logger(self, msg):
'''
write the log with local writer
:param msg:
:return:
'''
msg_now = f"{self.now} {msg.get('level')} <{msg.get('process')}> [{msg.get('logger_name')}] {msg.get('msg')}"
self.log_output(msg_now, msg)
dir = f"./data/log/"
if not os.path.exists(dir):
try:
os.makedirs(dir)
except Exception as error:
pass
full_name = f"{dir}/log.log"
try:
with open(full_name, 'a+') as f:
f.write(f"{msg_now}\n")
f.close()
except Exception as e:
with open(full_name, 'a+', encoding='utf-8') as f:
f.write(f"{msg_now}\n")
f.close()
@property
def now(self):
'''
get datetime now to str
:return: time now str
'''
current_time = time.time()
local_time = time.localtime(current_time)
time_temp = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
secs = (current_time - int(current_time)) * 1000
timenow = "%s %03d" % (time_temp, secs)
return timenow
def msg_write(self, msg):
'''
write to local or not
:param msg:
:return:
'''
if self.bus_client is None:
self.bus_client = BusClient()
if self.local_logger:
self.write_local_logger(msg)
else:
if msg.get('level') == 'debug' and not self.debug_:
return
self.log.put(msg)
class Program(object):
def __init__(self):
self.bus_client = None
self.case_name = ""
def _start_bus(self, local=True):
'''
_start_bus : start bus server with default config
:param local: bool, default is True
:return: None
'''
if local:
return
elif self.args.bus_server_port:
self.bus_server = BusServer(self.args.bus_server_port)
else:
self.bus_server = BusServer()
self.bus_server.start()
time.sleep(0.5)
def _start_loader(self, count: int, bus_client: BusClient):
'''
_start_loader : start loader (count)
:param count: loader count
:param bus_client: bus client -> BusClient (connect to bus server)
:return: None
'''
for x in range(count):
loader = Loader(bus_client, self.loader_recorder_lock, self.args)
loader.start()
def _start_runner(self, count: int, log_dir: str, bus_client: BusClient):
'''
_start_runner : start runner (count)
:param count: runner count
:param log_dir: log
:return:
'''
for x in range(count):
runner = Runner(log_dir, bus_client, self.multi_process_locks, self.args)
runner.start()
def _start_recorder(self, bus_client: BusClient, count: int = 1, log_dir: str = "", time_str: str = ""):
'''
start recorder
:param bus_client:
:param count:
:param log_dir:
:param time_str:
:return:
'''
recorder = Recorder(bus_client, count, self.case_name, time_str, log_dir, self.args.report_template,
self.loader_recorder_lock, self.args)
recorder.start()
def _init_logging_module(self, args):
'''
init logging module
:param args:
:return:
'''
logging.basicConfig(level=logging.DEBUG if not args.debug else logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s')
pass
def _init_system_logger(self, log_dir: str, bus_client: BusClient):
'''
init system logger
:param log_dir:
:param bus_client:
:return:
'''
log = Logger(self.case_name, self.time_str, log_dir, bus_client, self.args)
log.start()
def _init_system_lock(self, args):
'''
generate some locker
:param args:
:return:
'''
self.loader_recorder_lock = m_lock()
self.multi_process_locks = [m_lock() for x in range(4)]
if self.bus_client.get_case_count().empty():
self.bus_client.get_case_count().put(0)
def _bus_client(self, args):
'''
init bus client
:param args:
:return:
'''
if isinstance(args.bus_server, list):
self.bus_client = BusClient(args.bus_server[1], args.bus_server[2], args.bus_server[0])
elif args.bus_server_port:
self.bus_client = BusClient(None, args.bus_server_port, None)
else:
self.bus_client = BusClient()
self.params_loader = self.bus_client.get_param()
def start_main(self):
self.params_loader.put(Signal("main", SIGNAL_START))
def stop_main(self):
self.params_loader.put(Signal("main", SIGNAL_STOP))
def put_loader_msg(self, args):
if args.case:
for arg in args.case:
self.params_loader.put({"file_name": arg})
def _start_web_server(self, args):
plugin_manager.start_web_server(args, self.bus_client)
def run(self, args):
try:
self.args = args
self._init_logging_module(args)
self.case_name = self.args.name
self.time_str = Utils.get_time_str()
self.case_log_dir = f"{args.log_dir}/{self.case_name}/{self.time_str}"
self.runner_count = args.runner_count if args.runner_count else 1
self.only_bus(args)
self._start_bus(local=args.bus_server if args.bus_server else False)
self._bus_client(args)
self._init_system_lock(args)
# this only in server bus
if not args.bus_server:
self._init_system_logger(args.log_dir, self.bus_client)
# only : module
if args.only_loader:
self._start_loader(1, self.bus_client)
elif args.only_runner:
self._start_runner(self.runner_count, self.case_log_dir, self.bus_client)
elif args.only_recorder:
self._start_recorder(self.bus_client, self.runner_count, self.case_log_dir, self.time_str)
else:
self._start_loader(1, self.bus_client)
self._start_runner(self.runner_count, self.case_log_dir, self.bus_client)
self._start_recorder(self.bus_client, self.runner_count, self.case_log_dir, self.time_str)
self.start_main()
self.put_loader_msg(args)
self.stop_main()
self._start_web_server(args)
self.wait_end_signal(args)
except KeyboardInterrupt as key_inter:
logger.error(key_inter)
except FailLoaderException as loader_inter:
logger.error(loader_inter)
except FailRecorderException as recorder_inter:
logger.error(recorder_inter)
except FailRunnerException as runner_inter:
logger.error(runner_inter)
except FailFrameworkException as frame_inter:
logger.error(frame_inter)
def only_bus(self, args):
if args.only_bus:
self._start_bus(local=args.bus_server if args.bus_server else False)
self._bus_client(args)
self._init_system_lock(args)
self._init_system_logger(args.log_dir, self.bus_client)
self._start_recorder(self.bus_client, self.runner_count, self.case_log_dir, self.time_str)
self.wait_end_signal(args)
sys.exit(0)
else:
logger.debug("not only bus mode")
def only_loader(self, args):
if args.only_loader:
pass
def wait_end_signal(self, args):
try:
time.sleep(2)
system_signal = self.bus_client.get_system()
while True:
if not args.console:
if not system_signal.empty():
self.signal = system_signal.get()
signal = self.signal.signal if isinstance(self.signal, Signal) else None
logger.info(f"program signal {SIGNAL_GROUP.get(signal) if signal in [x for x in range(20, 30)] else None}")
# check the end signal from recorder to main
if signal == SIGNAL_RECORD_END or signal == SIGNAL_STOP:
if not args.local_logger:
while True:
if not system_signal.empty():
signal_logger = system_signal.get()
signal_logger = signal_logger.signal if isinstance(signal_logger,
Signal) else None
logger.info(f"program signal {SIGNAL_GROUP.get(signal_logger) if signal_logger in [x for x in range(20, 30)] else None}")
# check the logger signal from logger to main
if signal_logger == SIGNAL_LOGGER_END:
logger.info("main -- stop")
system_signal.put(Signal("main", SIGNAL_RECORD_END))
system_signal.put(Signal("main", SIGNAL_LOGGER_END))
break
# if use local logger, just end the main program
else:
logger.info("main -- stop")
system_signal.put(Signal("main", SIGNAL_RECORD_END))
system_signal.put(Signal("main", SIGNAL_LOGGER_END))
break
time.sleep(0.1)
else:
cmd = input(f"haf-{PLATFORM_VERSION}# ")
if self._run_cmd(cmd):
break
if args.only_bus:
logger.info("main - wait for other process end !")
time.sleep(3)
time.sleep(0.1)
except KeyboardInterrupt as key_inter:
self.params_loader.put(Signal("main", SIGNAL_STOP))
# TODO: Here need CMDER to support cmd command ...
def _run_cmd(self, cmd):
if cmd == "rerun" or cmd == "r":
result = self._rerun()
elif cmd == "version" or cmd == "v":
result = self._version()
elif cmd == "help" or cmd == "h":
result = self._help()
elif cmd == "name" or cmd == "n":
result = self._case_name()
elif cmd == "exit" or cmd == "e":
result = self._exit()
elif cmd == "summary" or cmd == "s":
result = self._summary()
else:
print("unsupported command!")
result = self._help()
return result
def _rerun(self):
case_handler = self.bus_client.get_case()
while not case_handler.empty():
case_handler.get()
self._start_runner(self.args.runner_count if self.args.runner_count else 1, self.case_log_dir, self.bus_client)
self._start_loader(1, self.bus_client)
self._start_recorder(self.bus_client, self.runner_count, self.case_log_dir, self.time_str)
self.start_main()
self.put_loader_msg(self.args)
self.stop_main()
return False
def _version(self):
print(BANNER_STRS)
return False
def _help(self):
help = f"""
haf-{PLATFORM_VERSION}#
# rerun / r rerun the input cases
# version / v version of haf
# help / h help information
# name / n case name of this test
# summary / s summary of this test
# exit / e exit
"""
print(help)
return False
def _summary(self):
result_main = self.bus_client.get_case_result_main()
if not result_main.empty():
results = result_main.get()
if isinstance(results, EndResult):
self.results = results
if hasattr(self, "results") and isinstance(self.results, EndResult):
pass
else:
self.results = EndResult()
result_summary = "|{:^8}|{:^8}|{:^8}|{:^8}|{:^8}|{:^25}|{:^25}|".format(self.results.passed,
self.results.failed, self.results.skip, \
self.results.error, self.results.all,
self.results.begin_time,
self.results.end_time)
print("--------------------------------------------------------------------------------------------------")
print(
"|--\33[32mPASS\33[0m--|--\33[31mFAIL\33[0m--|--\33[37mSKIP\33[0m--|--\33[35mERROR\33[0m-|---\33[36mALL\33[0m--|----------\33[36mBegin\33[0m----------|-----------\33[36mEnd\33[0m-----------|")
print(result_summary)
print("--------------------------------------------------------------------------------------------------")
def _case_name(self):
print(self.case_name)
return False
def _exit(self):
logger.info("main -- stop, wait for bus end")
print(BANNER_STRS_EXIT)
return True
def __init__(self):
super().__init__()
self.bus_client = BusClient()
class Logger(Process):
'''
Logger
'''
def __init__(self, case_name: str, time_str: str, log_dir: str,
bus_client: BusClient, args: tuple):
super().__init__()
self.daemon = True
self.bus_client = bus_client
self.case_name = case_name
self.log_dir = log_dir
self.loggers = {}
self.args = args
self.time_str = time_str
def reconnect_bus(self):
'''
need reconnect bus by self.bus's infos
:return:
'''
self.bus_client = BusClient(self.bus_client.domain,
self.bus_client.port,
self.bus_client.auth_key)
def run(self):
self.reconnect_bus()
if hasattr(self.args, "debug") and self.args.debug:
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s')
else:
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s %(message)s')
log_home = f"{self.log_dir}/{self.case_name}/{self.time_str}"
if not os.path.exists(log_home):
os.makedirs(log_home)
# here delete BusClient(), using input bus_clients
# self.bus_client = BusClient()
try:
log_queue = self.bus_client.get_log()
logger_end = self.bus_client.get_logger_end()
while True:
if not logger_end.empty():
logger_signal = logger_end.get()
# check logger end singal to stop logger, from recorder to logger
if isinstance(
logger_signal, Signal
) and logger_signal.signal == SIGNAL_LOGGER_END:
while True:
if log_queue.empty():
break
log = log_queue.get()
self.log_handler(log)
self.end_handler()
break
elif log_queue.empty():
time.sleep(0.001)
continue
else:
log = log_queue.get()
self.log_handler(log)
except KeyboardInterrupt as key_e:
print(BANNER_STRS_EXIT)
def split_log(self, log):
'''
split origin log msg
:param log:
:return:
'''
try:
return log.rsplit("$%", 2)
except Exception as ee:
return f"log$%error$%{log}"
def log_print(self, log):
'''
to print the log with format
:param log:
:return:
'''
if self.args.nout:
return
logger_name = log.get("logger_name")
level = log.get("level")
msg_origin = log.get("msg")
process = log.get("process")
if process not in self.loggers:
self.loggers[process] = logging.getLogger(logger_name)
logger = self.loggers.get(process)
if level == "debug":
msg = f"<{process}> [{logger_name}] {msg_origin}"
if hasattr(self.args, "debug") and self.args.debug:
logger.debug(msg)
elif level == "info":
msg = f"<{process}> [{logger_name}] {msg_origin}"
logger.info(msg)
elif level == "warning":
msg = f"{msg_origin}"
logger.warning(
f"<{process}> [{logger_name}]\33[33m ####################################### \33[0m"
)
logger.warning(f"<{process}> [{logger_name}] \33[33m{msg}\33[0m")
logger.warning(
f"<{process}> [{logger_name}]\33[33m ####################################### \33[0m"
)
elif level == "error":
msg = f"{msg_origin}"
logger.error(
f"<{process}> [{logger_name}]\33[31m >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> \33[0m"
)
logger.error(f"<{process}> [{logger_name}]\33[31m | {msg}\33[0m")
logger.error(
f"<{process}> [{logger_name}]\33[31m <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< \33[0m"
)
elif level == "critical":
logger.critical(f"\33[5m{log}\33[0m")
def log_handler(self, log):
'''
handler log here
:param log:
:return:
'''
self.log_print(log)
log = log.get("msg")
try:
temp1, temp2, msg = self.split_log(log)
except:
temp1 = "error"
temp2 = "error"
msg = log
if "loader" in temp2:
self.loader_handler(temp1, msg)
elif "runner" in temp2:
self.runner_handler(temp1, msg)
elif "recorder" in temp2:
self.recorder_handler(temp1, msg)
elif "system" in temp2:
self.system_handler(temp1, temp2, msg)
elif "error" in temp2:
self.error_handler(temp1, msg)
else:
self.case_handler(temp1, temp2, msg)
def loader_handler(self, temp1, msg):
self.write("loader", temp1, msg)
def runner_handler(self, temp1, msg):
self.write("runner", temp1, msg)
def case_handler(self, temp1, temp2, msg):
self.write(temp1, temp2, msg)
def recorder_handler(self, temp1, msg):
self.write("recorder", temp1, msg)
def system_handler(self, temp1, temp2, msg):
self.write("system", f"{temp1}.{temp2}", msg)
def error_handler(self, temp1, msg):
self.write("error", temp1, msg)
def write(self, dir, filename, msg):
'''
write log to filename's file
:param dir:
:param filename:
:param msg:
:return:
'''
msg = f"{self.now}{msg}\n"
dir = f"{self.log_dir}/{self.case_name}/{self.time_str}/{dir}"
if not os.path.exists(dir):
os.makedirs(dir)
full_name = f"{dir}/{filename}.log"
try:
with open(full_name, 'a+') as f:
f.write(msg)
f.close()
except Exception as e:
with open(full_name, 'a+', encoding='utf-8') as f:
f.write(msg)
f.close()
@property
def now(self):
'''
get datetime now to str
:return: time now str
'''
current_time = time.time()
local_time = time.localtime(current_time)
time_temp = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
secs = (current_time - int(current_time)) * 1000
timenow = "%s %03d" % (time_temp, secs)
return timenow
def end_handler(self):
'''
when logger end, send signal logger end to main
:return:
'''
logger_handler = self.bus_client.get_system()
logger_handler.put(Signal(self.pid, SIGNAL_LOGGER_END))
def report_app() -> str:
ResultResource().get()
report_stream = Jinja2Report.report_online_app(globalenv.get_global("results"))
return report_stream
@app.route("/report-web")
def report_web() -> str:
ResultResource().get()
report_stream = Jinja2Report.report_online_app(globalenv.get_global("results"))
return report_stream
@app.route("/status")
def status() -> str:
return '{"status": "ok"}'
def abort_if_not_exist():
abort(404, message="404")
def web_server(bus_client: BusClient):
bus_client = BusClient(bus_client.domain, bus_client.port, bus_client.auth_key)
api.add_resource(LoaderResource, "/loader")
api.add_resource(RunnerResource, "/runner")
api.add_resource(ResultResource, "/result")
api.add_resource(DocResource, "/doc")
app.run(debug=False, port=WEB_SERVER_PORT)
if __name__ == "__main__":
web_server(BusClient())
