def _waiting_output_str(manager: LogManager, /) -> None:
pid_cur = current_process().pid
running = True
printer = _SysOutPrinter(manager)
_print_empty(printer, pid_cur, manager)
try:
while running:
try:
erg = manager.get_from_queue()
except queue.Empty:
_print_empty(printer, pid_cur, manager)
else:
if isinstance(erg, tuple) and len(erg) == 2:
printer.print_to_console(erg[0], erg[1])
running = manager.run or not manager.queue_empty
out_msg = "_OutputWaitingProcess [done]"
act_obj = len(manager.sync_object_manager.managed_dict) - 1
act_obj -= manager.manager_counter.done_object_counter.value
if act_obj:
out_msg = f"{out_msg} unfinished objects: {act_obj}"
printer.print_to_console(
SyncOutMsg(s_id="_OutputWaitingProcess_{0}_{1}".format(
pid_cur, manager.id_cnt),
msg=out_msg,
status=100.0,
done=True), format_message_time)
except Exception as exc:
manager.set_manager_error()
raise exc
finally:
manager.zero_status()
def _print_empty(printer: _SysOutPrinter, pid_cur: Optional[int],
manager: LogManager, /) -> None:
printer.print_to_console(
SyncOutMsg(s_id="_OutputWaitingProcess_{0}_{1}".format(
pid_cur, manager.id_cnt),
msg="",
status=0.0,
done=False), format_message_time)
def logger_print_con(sync_out: SyncStdoutInterface, object_name: str,
message: str, status: float, done: bool, /) -> None:
sync_out.print_message(
SyncOutMsg(s_id="{0}_{1}".format(object_name,
current_process().pid),
msg=message,
status=status,
done=done), format_message_time)
def print_message_simple(self, message: str, /) -> None:
if self._check_closed():
raise SyncStdoutError("The logger was already closed!")
if self.__general.global_manager.manager_error:
_increase_error(self.__logging_stats)
sys.__stdout__.write(
"ERROR occurred in output process (printing message simple)!")
sys.__stdout__.flush()
else:
self.__general.global_manager.writer_to_queue(
(SyncOutMsg(s_id="SyncStdout_{0}".format(self.__general.g_id),
msg=message,
status=0.0,
done=False), _simple_formatter))
def __init__(self, stdout_args: SyncStdoutArgs,
ctx_local: multiprocessing.context.SpawnContext,
manager: LogManager,
pr_list: List[multiprocessing.context.SpawnProcess],
/) -> None:
super().__init__()
create_dirs_rec(stdout_args.working_dir)
if manager.run_with_update(True):
pr_list.append(
ctx_local.Process(target=_waiting_output_str,
args=(manager, )))
pr_list[-1].start()
self.__logger_data = _LoggerData(
lock=ctx_local.RLock(),
queue=ctx_local.Queue(),
running=CustomProcessValueWrapper[int](ctx_local, 1),
finished=CustomProcessValueWrapper[int](ctx_local, 1))
pr_list.append(
ctx_local.Process(target=_logger_process,
args=(self.__logger_data,
stdout_args.working_dir,
stdout_args.step_name)))
pr_list[-1].start()
self.__stats_lock = ctx_local.RLock()
self.__logging_stats: _LoggingStats = _LoggingStats(
errors_cnt=CustomProcessValueWrapper[int](ctx_local, 0),
info_cnt=CustomProcessValueWrapper[int](ctx_local, 0),
warn_cnt=CustomProcessValueWrapper[int](ctx_local, 0),
other_cnt=CustomProcessValueWrapper[int](ctx_local, 0))
self.__general: _General = _General(g_id=manager.logger_cnt,
step=stdout_args.step_name,
working_dir=str(
stdout_args.working_dir),
global_manager=manager)
self.__stats: CustomManagedDictWrapper[str, JsonStatsContainerIntFloatList] = \
CustomManagedDictWrapper[str, JsonStatsContainerIntFloatList](ctx_local)
self.__end_con: _EndConD = _EndConD(
flush_stats=CustomProcessValueWrapper[int](ctx_local, 1),
flush_log=CustomProcessValueWrapper[int](ctx_local, 1))
if self.__general.global_manager.manager_error:
_increase_error(self.__logging_stats)
sys.__stdout__.write(
"ERROR occurred in output process (init step)!")
sys.__stdout__.flush()
else:
manager.writer_to_queue((SyncOutMsg(
s_id="SyncStdout_{0}".format(self.__general.g_id),
msg="SyncStdout_{0} started!".format(self.__general.g_id),
status=0.0,
done=False), format_message_time))
def print_message(self, message: SyncOutMsg,
format_func: Callable[[str], List[str]], /) -> None:
if self._check_closed():
raise SyncStdoutError("The logger was already closed!")
message.s_id = "SyncStdout_{0}_{1}".format(self.__general.g_id,
message.s_id)
if self.__general.global_manager.manager_error:
_increase_error(self.__logging_stats)
sys.__stdout__.write(
"ERROR occurred in output process (printing message)!")
sys.__stdout__.flush()
else:
self.__general.global_manager.writer_to_queue(
(message, format_func))
def close_logger(self) -> None:
@final
@dataclass
class _LStatus:
flush_log: int
flush_stats: int
msg = ""
with self.__stats_lock:
erg_flush = _LStatus(flush_log=self.__end_con.flush_log.value,
flush_stats=self.__end_con.flush_stats.value)
self.__end_con.flush_log.value = 0
self.__end_con.flush_stats.value = 0
if erg_flush.flush_log:
msg += "The logger log_stats were not flushed!\n"
if erg_flush.flush_stats:
msg += "The logger stats were not flushed!\n"
if not (erg_flush.flush_log or erg_flush.flush_stats):
msg += "The logger was closed correctly!\n"
if self.__general.global_manager.manager_error:
_increase_error(self.__logging_stats)
sys.__stdout__.write(
"ERROR occurred in output process (closing step)!")
sys.__stdout__.flush()
else:
self.__general.global_manager.writer_to_queue(
(SyncOutMsg(s_id="SyncStdout_{0}".format(self.__general.g_id),
msg=msg,
status=100.0,
done=True), format_message_time))
_ = self.__general.global_manager.run_with_update(False)
with self.__logger_data.lock:
self.__logger_data.running.value = 0
self._join_logger()
self.__general.global_manager.join()
def flush_log_stats(self) -> None:
with self.__stats_lock:
flushed = self.__end_con.flush_log.value
self.__end_con.flush_log.value = 0
if flushed:
msg = "Step {0}: Errors: {1}; Warnings: {2}; Info: {3}; Other: {4}".format(
self.__general.step, self.__logging_stats.errors_cnt.value,
self.__logging_stats.warn_cnt.value,
self.__logging_stats.info_cnt.value,
self.__logging_stats.other_cnt.value)
if self.__general.global_manager.manager_error:
_increase_error(self.__logging_stats)
sys.__stdout__.write(
"ERROR occurred in output process (flushing step)!")
sys.__stdout__.flush()
else:
self.__general.global_manager.writer_to_queue((SyncOutMsg(
s_id="SyncStdout_{0}".format(self.__general.g_id),
msg=msg,
status=50.0,
done=False), format_message_time))