def prepare_global_logging(serialization_dir: str, file_friendly_logging: bool) -> None: """ This function configures 3 global logging attributes - streaming stdout and stderr to a file as well as the terminal, setting the formatting for the python logging library and setting the interval frequency for the Tqdm progress bar. Note that this function does not set the logging level, which is set in ``allennlp/run.py``. Parameters ---------- serializezation_dir : ``str``, required. The directory to stream logs to. file_friendly_logging : ``bool``, required. Whether logs should clean the output to prevent carridge returns (used to update progress bars on a single terminal line). """ Tqdm.set_slower_interval(file_friendly_logging) std_out_file = os.path.join(serialization_dir, "stdout.log") sys.stdout = TeeLogger( std_out_file, # type: ignore sys.stdout, file_friendly_logging) sys.stderr = TeeLogger( os.path.join(serialization_dir, "stderr.log"), # type: ignore sys.stderr, file_friendly_logging) stdout_handler = logging.FileHandler(std_out_file) stdout_handler.setFormatter( logging.Formatter( '%(asctime)s - %(levelname)s - %(name)s - %(message)s')) logging.getLogger().addHandler(stdout_handler)
def prepare_global_logging(serialization_dir: str, file_friendly_logging: bool) -> None: """ This function configures 3 global logging attributes - streaming stdout and stderr to a file as well as the terminal, setting the formatting for the python logging library and setting the interval frequency for the Tqdm progress bar. Note that this function does not set the logging level, which is set in ``allennlp/run.py``. Parameters ---------- serializezation_dir : ``str``, required. The directory to stream logs to. file_friendly_logging : ``bool``, required. Whether logs should clean the output to prevent carridge returns (used to update progress bars on a single terminal line). """ Tqdm.set_slower_interval(file_friendly_logging) std_out_file = os.path.join(serialization_dir, "stdout.log") sys.stdout = TeeLogger(std_out_file, # type: ignore sys.stdout, file_friendly_logging) sys.stderr = TeeLogger(os.path.join(serialization_dir, "stderr.log"), # type: ignore sys.stderr, file_friendly_logging) stdout_handler = logging.FileHandler(std_out_file) stdout_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(name)s - %(message)s')) logging.getLogger().addHandler(stdout_handler)
def prepare_global_logging(serialization_dir: str, file_friendly_logging: bool) -> logging.FileHandler: """ This function configures 3 global logging attributes - streaming stdout and stderr to a file as well as the terminal, setting the formatting for the python logging library and setting the interval frequency for the Tqdm progress bar. Note that this function does not set the logging level, which is set in ``allennlp/run.py``. Parameters ---------- serialization_dir : ``str``, required. The directory to stream logs to. file_friendly_logging : ``bool``, required. Whether logs should clean the output to prevent carriage returns (used to update progress bars on a single terminal line). This option is typically only used if you are running in an environment without a terminal. Returns ------- ``logging.FileHandler`` A logging file handler that can later be closed and removed from the global logger. """ # If we don't have a terminal as stdout, # force tqdm to be nicer. if not sys.stdout.isatty(): file_friendly_logging = True Tqdm.set_slower_interval(file_friendly_logging) std_out_file = os.path.join(serialization_dir, "stdout.log") sys.stdout = TeeLogger( std_out_file, # type: ignore sys.stdout, file_friendly_logging) sys.stderr = TeeLogger( os.path.join(serialization_dir, "stderr.log"), # type: ignore sys.stderr, file_friendly_logging) stdout_handler = logging.FileHandler(std_out_file) stdout_handler.setFormatter( logging.Formatter( '%(asctime)s - %(levelname)s - %(name)s - %(message)s')) logging.getLogger().addHandler(stdout_handler) return stdout_handler
def prepare_global_logging(serialization_dir: str, file_friendly_logging: bool, rank: int = 0, world_size: int = 1) -> None: # If we don't have a terminal as stdout, # force tqdm to be nicer. if not sys.stdout.isatty(): file_friendly_logging = True Tqdm.set_slower_interval(file_friendly_logging) # Handlers for stdout/err logging output_stream_log_handler = logging.StreamHandler(sys.stdout) error_stream_log_handler = logging.StreamHandler(sys.stderr) if world_size == 1: # This case is not distributed training and hence will stick to the older # log file names output_file_log_handler = logging.FileHandler( filename=os.path.join(serialization_dir, "stdout.log")) error_file_log_handler = logging.FileHandler( filename=os.path.join(serialization_dir, "stderr.log")) else: # Create log files with worker ids output_file_log_handler = logging.FileHandler(filename=os.path.join( serialization_dir, f"stdout_worker{rank}.log")) error_file_log_handler = logging.FileHandler(filename=os.path.join( serialization_dir, f"stderr_worker{rank}.log")) # This adds the worker's rank to messages being logged to files. # This will help when combining multiple worker log files using `less` command. worker_filter = WorkerLogFilter(rank) output_file_log_handler.addFilter(worker_filter) error_file_log_handler.addFilter(worker_filter) formatter = logging.Formatter( "%(asctime)s - %(levelname)s - %(name)s - %(message)s") root_logger = logging.getLogger() # Remove the already set stream handler in root logger. # Not doing this will result in duplicate log messages # printed in the console if len(root_logger.handlers) > 0: for handler in root_logger.handlers: root_logger.removeHandler(handler) # file handlers need to be handled for tqdm's \r char file_friendly_log_filter = FileFriendlyLogFilter() if os.environ.get("ALLENNLP_DEBUG"): LEVEL = logging.DEBUG else: LEVEL = logging.INFO if rank == 0: # stdout/stderr handlers are added only for the # master worker. This is to avoid cluttering the console # screen with too many log messages from all workers. output_stream_log_handler.setFormatter(formatter) error_stream_log_handler.setFormatter(formatter) output_stream_log_handler.setLevel(LEVEL) error_stream_log_handler.setLevel(logging.ERROR) if file_friendly_logging: output_stream_log_handler.addFilter(file_friendly_log_filter) error_stream_log_handler.addFilter(file_friendly_log_filter) root_logger.addHandler(output_stream_log_handler) root_logger.addHandler(error_stream_log_handler) output_file_log_handler.addFilter(file_friendly_log_filter) error_file_log_handler.addFilter(file_friendly_log_filter) output_file_log_handler.setFormatter(formatter) error_file_log_handler.setFormatter(formatter) output_file_log_handler.setLevel(LEVEL) error_file_log_handler.setLevel(logging.ERROR) root_logger.addHandler(output_file_log_handler) root_logger.addHandler(error_file_log_handler) root_logger.setLevel(LEVEL)
def prepare_global_logging(serialization_dir: str, file_friendly_logging: bool, rank: int = 0, world_size: int = 1) -> None: # If we don't have a terminal as stdout, # force tqdm to be nicer. if not sys.stdout.isatty(): file_friendly_logging = True Tqdm.set_slower_interval(file_friendly_logging) stdout_file: str stderr_file: str worker_filter: Optional[WorkerLogFilter] = None if world_size == 1: # This case is not distributed training and hence will stick to the older # log file names stdout_file = os.path.join(serialization_dir, "stdout.log") stderr_file = os.path.join(serialization_dir, "stderr.log") else: # Create log files with worker ids stdout_file = os.path.join(serialization_dir, f"stdout_worker{rank}.log") stderr_file = os.path.join(serialization_dir, f"stderr_worker{rank}.log") # This adds the worker's rank to messages being logged to files. # This will help when combining multiple worker log files using `less` command. worker_filter = WorkerLogFilter(rank) # Patch stdout/err. stdout_patch = TeeHandler( stdout_file, sys.stdout, file_friendly_terminal_output=file_friendly_logging, silent=rank != 0, # don't print to terminal from non-master workers. ) sys.stdout = stdout_patch # type: ignore stderr_patch = TeeHandler( stderr_file, sys.stderr, file_friendly_terminal_output=file_friendly_logging, silent=rank != 0, # don't print to terminal from non-master workers. ) sys.stderr = stderr_patch # type: ignore # Handlers for stdout/err logging output_handler = logging.StreamHandler(sys.stdout) error_handler = logging.StreamHandler(sys.stderr) if worker_filter is not None: output_handler.addFilter(worker_filter) error_handler.addFilter(worker_filter) root_logger = logging.getLogger() # Remove the already set stream handler in root logger. # Not doing this will result in duplicate log messages # printed in the console if len(root_logger.handlers) > 0: for handler in root_logger.handlers: root_logger.removeHandler(handler) formatter = logging.Formatter( "%(asctime)s - %(levelname)s - %(name)s - %(message)s") output_handler.setFormatter(formatter) error_handler.setFormatter(formatter) if os.environ.get("ALLENNLP_DEBUG"): LEVEL = logging.DEBUG else: level_name = os.environ.get("ALLENNLP_LOG_LEVEL") LEVEL = logging._nameToLevel.get(level_name, logging.INFO) output_handler.setLevel(LEVEL) error_handler.setLevel(logging.ERROR) # filter out everything at the ERROR or higher level for output stream # so that error messages don't appear twice in the logs. output_handler.addFilter(ErrorFilter()) root_logger.addHandler(output_handler) root_logger.addHandler(error_handler) root_logger.setLevel(LEVEL)