def prepare_global_logging(serialization_dir: str,
file_friendly_logging: bool) -> None:
"""
This function configures 3 global logging attributes - streaming stdout and stderr
to a file as well as the terminal, setting the formatting for the python logging
library and setting the interval frequency for the Tqdm progress bar.
Note that this function does not set the logging level, which is set in ``allennlp/run.py``.
Parameters
----------
serializezation_dir : ``str``, required.
The directory to stream logs to.
file_friendly_logging : ``bool``, required.
Whether logs should clean the output to prevent carridge returns
(used to update progress bars on a single terminal line).
"""
Tqdm.set_slower_interval(file_friendly_logging)
std_out_file = os.path.join(serialization_dir, "stdout.log")
sys.stdout = TeeLogger(
std_out_file, # type: ignore
sys.stdout,
file_friendly_logging)
sys.stderr = TeeLogger(
os.path.join(serialization_dir, "stderr.log"), # type: ignore
sys.stderr,
file_friendly_logging)
stdout_handler = logging.FileHandler(std_out_file)
stdout_handler.setFormatter(
logging.Formatter(
'%(asctime)s - %(levelname)s - %(name)s - %(message)s'))
logging.getLogger().addHandler(stdout_handler)
def prepare_global_logging(serialization_dir: str, file_friendly_logging: bool) -> None:
"""
This function configures 3 global logging attributes - streaming stdout and stderr
to a file as well as the terminal, setting the formatting for the python logging
library and setting the interval frequency for the Tqdm progress bar.
Note that this function does not set the logging level, which is set in ``allennlp/run.py``.
Parameters
----------
serializezation_dir : ``str``, required.
The directory to stream logs to.
file_friendly_logging : ``bool``, required.
Whether logs should clean the output to prevent carridge returns
(used to update progress bars on a single terminal line).
"""
Tqdm.set_slower_interval(file_friendly_logging)
std_out_file = os.path.join(serialization_dir, "stdout.log")
sys.stdout = TeeLogger(std_out_file, # type: ignore
sys.stdout,
file_friendly_logging)
sys.stderr = TeeLogger(os.path.join(serialization_dir, "stderr.log"), # type: ignore
sys.stderr,
file_friendly_logging)
stdout_handler = logging.FileHandler(std_out_file)
stdout_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(name)s - %(message)s'))
logging.getLogger().addHandler(stdout_handler)
def prepare_global_logging(serialization_dir: str,
file_friendly_logging: bool) -> logging.FileHandler:
"""
This function configures 3 global logging attributes - streaming stdout and stderr
to a file as well as the terminal, setting the formatting for the python logging
library and setting the interval frequency for the Tqdm progress bar.
Note that this function does not set the logging level, which is set in ``allennlp/run.py``.
Parameters
----------
serialization_dir : ``str``, required.
The directory to stream logs to.
file_friendly_logging : ``bool``, required.
Whether logs should clean the output to prevent carriage returns
(used to update progress bars on a single terminal line). This
option is typically only used if you are running in an environment
without a terminal.
Returns
-------
``logging.FileHandler``
A logging file handler that can later be closed and removed from the global logger.
"""
# If we don't have a terminal as stdout,
# force tqdm to be nicer.
if not sys.stdout.isatty():
file_friendly_logging = True
Tqdm.set_slower_interval(file_friendly_logging)
std_out_file = os.path.join(serialization_dir, "stdout.log")
sys.stdout = TeeLogger(
std_out_file, # type: ignore
sys.stdout,
file_friendly_logging)
sys.stderr = TeeLogger(
os.path.join(serialization_dir, "stderr.log"), # type: ignore
sys.stderr,
file_friendly_logging)
stdout_handler = logging.FileHandler(std_out_file)
stdout_handler.setFormatter(
logging.Formatter(
'%(asctime)s - %(levelname)s - %(name)s - %(message)s'))
logging.getLogger().addHandler(stdout_handler)
return stdout_handler
def prepare_global_logging(serialization_dir: str,
file_friendly_logging: bool,
rank: int = 0,
world_size: int = 1) -> None:
# If we don't have a terminal as stdout,
# force tqdm to be nicer.
if not sys.stdout.isatty():
file_friendly_logging = True
Tqdm.set_slower_interval(file_friendly_logging)
# Handlers for stdout/err logging
output_stream_log_handler = logging.StreamHandler(sys.stdout)
error_stream_log_handler = logging.StreamHandler(sys.stderr)
if world_size == 1:
# This case is not distributed training and hence will stick to the older
# log file names
output_file_log_handler = logging.FileHandler(
filename=os.path.join(serialization_dir, "stdout.log"))
error_file_log_handler = logging.FileHandler(
filename=os.path.join(serialization_dir, "stderr.log"))
else:
# Create log files with worker ids
output_file_log_handler = logging.FileHandler(filename=os.path.join(
serialization_dir, f"stdout_worker{rank}.log"))
error_file_log_handler = logging.FileHandler(filename=os.path.join(
serialization_dir, f"stderr_worker{rank}.log"))
# This adds the worker's rank to messages being logged to files.
# This will help when combining multiple worker log files using `less` command.
worker_filter = WorkerLogFilter(rank)
output_file_log_handler.addFilter(worker_filter)
error_file_log_handler.addFilter(worker_filter)
formatter = logging.Formatter(
"%(asctime)s - %(levelname)s - %(name)s - %(message)s")
root_logger = logging.getLogger()
# Remove the already set stream handler in root logger.
# Not doing this will result in duplicate log messages
# printed in the console
if len(root_logger.handlers) > 0:
for handler in root_logger.handlers:
root_logger.removeHandler(handler)
# file handlers need to be handled for tqdm's \r char
file_friendly_log_filter = FileFriendlyLogFilter()
if os.environ.get("ALLENNLP_DEBUG"):
LEVEL = logging.DEBUG
else:
LEVEL = logging.INFO
if rank == 0:
# stdout/stderr handlers are added only for the
# master worker. This is to avoid cluttering the console
# screen with too many log messages from all workers.
output_stream_log_handler.setFormatter(formatter)
error_stream_log_handler.setFormatter(formatter)
output_stream_log_handler.setLevel(LEVEL)
error_stream_log_handler.setLevel(logging.ERROR)
if file_friendly_logging:
output_stream_log_handler.addFilter(file_friendly_log_filter)
error_stream_log_handler.addFilter(file_friendly_log_filter)
root_logger.addHandler(output_stream_log_handler)
root_logger.addHandler(error_stream_log_handler)
output_file_log_handler.addFilter(file_friendly_log_filter)
error_file_log_handler.addFilter(file_friendly_log_filter)
output_file_log_handler.setFormatter(formatter)
error_file_log_handler.setFormatter(formatter)
output_file_log_handler.setLevel(LEVEL)
error_file_log_handler.setLevel(logging.ERROR)
root_logger.addHandler(output_file_log_handler)
root_logger.addHandler(error_file_log_handler)
root_logger.setLevel(LEVEL)
def prepare_global_logging(serialization_dir: str,
file_friendly_logging: bool,
rank: int = 0,
world_size: int = 1) -> None:
# If we don't have a terminal as stdout,
# force tqdm to be nicer.
if not sys.stdout.isatty():
file_friendly_logging = True
Tqdm.set_slower_interval(file_friendly_logging)
stdout_file: str
stderr_file: str
worker_filter: Optional[WorkerLogFilter] = None
if world_size == 1:
# This case is not distributed training and hence will stick to the older
# log file names
stdout_file = os.path.join(serialization_dir, "stdout.log")
stderr_file = os.path.join(serialization_dir, "stderr.log")
else:
# Create log files with worker ids
stdout_file = os.path.join(serialization_dir,
f"stdout_worker{rank}.log")
stderr_file = os.path.join(serialization_dir,
f"stderr_worker{rank}.log")
# This adds the worker's rank to messages being logged to files.
# This will help when combining multiple worker log files using `less` command.
worker_filter = WorkerLogFilter(rank)
# Patch stdout/err.
stdout_patch = TeeHandler(
stdout_file,
sys.stdout,
file_friendly_terminal_output=file_friendly_logging,
silent=rank != 0, # don't print to terminal from non-master workers.
)
sys.stdout = stdout_patch # type: ignore
stderr_patch = TeeHandler(
stderr_file,
sys.stderr,
file_friendly_terminal_output=file_friendly_logging,
silent=rank != 0, # don't print to terminal from non-master workers.
)
sys.stderr = stderr_patch # type: ignore
# Handlers for stdout/err logging
output_handler = logging.StreamHandler(sys.stdout)
error_handler = logging.StreamHandler(sys.stderr)
if worker_filter is not None:
output_handler.addFilter(worker_filter)
error_handler.addFilter(worker_filter)
root_logger = logging.getLogger()
# Remove the already set stream handler in root logger.
# Not doing this will result in duplicate log messages
# printed in the console
if len(root_logger.handlers) > 0:
for handler in root_logger.handlers:
root_logger.removeHandler(handler)
formatter = logging.Formatter(
"%(asctime)s - %(levelname)s - %(name)s - %(message)s")
output_handler.setFormatter(formatter)
error_handler.setFormatter(formatter)
if os.environ.get("ALLENNLP_DEBUG"):
LEVEL = logging.DEBUG
else:
level_name = os.environ.get("ALLENNLP_LOG_LEVEL")
LEVEL = logging._nameToLevel.get(level_name, logging.INFO)
output_handler.setLevel(LEVEL)
error_handler.setLevel(logging.ERROR)
# filter out everything at the ERROR or higher level for output stream
# so that error messages don't appear twice in the logs.
output_handler.addFilter(ErrorFilter())
root_logger.addHandler(output_handler)
root_logger.addHandler(error_handler)
root_logger.setLevel(LEVEL)
