def prepare_global_logging(params) -> None:
"""
This function configures 3 global logging attributes - streaming stdout and stderr
to a file as well as the terminal, setting the formatting for the python logging
library and setting the interval frequency for the Tqdm progress bar.
Note that this function does not set the logging level, which is set in ``allennlp/run.py``.
Parameters
----------
serializezation_dir : ``str``, required.
The directory to stream logs to.
file_friendly_logging : ``bool``, required.
Whether logs should clean the output to prevent carridge returns
(used to update progress bars on a single terminal line).
"""
serialization_dir = params['serialization_dir']
file_friendly_logging = params['file_friendly_logging']
Tqdm.set_slower_interval(file_friendly_logging)
std_out_file = os.path.join(serialization_dir, "stdout.log")
sys.stdout = TeeLogger(std_out_file, # type: ignore
sys.stdout,
file_friendly_logging)
sys.stderr = TeeLogger(os.path.join(serialization_dir, "stderr.log"), # type: ignore
sys.stderr,
file_friendly_logging)
logging.init_logger(log_file=std_out_file)
import json
import warnings
import argparse
from xlamr_stog.utils import logging
from xlamr_stog.utils.params import Params, remove_pretrained_embedding_params
from xlamr_stog import models as Models
from xlamr_stog.data.dataset_builder import dataset_from_params, iterator_from_params
from xlamr_stog.data.vocabulary import Vocabulary
from xlamr_stog.training.trainer import Trainer
from xlamr_stog.utils import environment
from xlamr_stog.utils.checks import ConfigurationError
from xlamr_stog.utils.archival import CONFIG_NAME, _DEFAULT_WEIGHTS, archive_model
from xlamr_stog.commands.evaluate import evaluate
from xlamr_stog.metrics import dump_metrics
logger = logging.init_logger()
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
warnings.filterwarnings("ignore")
def serialize_sets(obj):
if isinstance(obj, set):
return list(obj)
return obj
def create_serialization_dir(params: Params) -> None:
"""
This function creates the serialization directory if it doesn't exist. If it already exists
and is non-empty, then it verifies that we're recovering from a training with an identical configuration.
Parameters
----------
"""
Adopted from AllenNLP:
https://github.com/allenai/allennlp/tree/v0.6.1/allennlp/common
Functions and exceptions for checking that
AllenNLP and its models are configured correctly.
"""
from torch import cuda
from xlamr_stog.utils import logging
logger = logging.init_logger() # pylint: disable=invalid-name
class ConfigurationError(Exception):
"""
The exception raised by any AllenNLP object when it's misconfigured
(e.g. missing properties, invalid properties, unknown properties).
"""
def __init__(self, message):
super(ConfigurationError, self).__init__()
self.message = message
def __str__(self):
return repr(self.message)
def log_pytorch_version_info():
import torch
logger.info("Pytorch version: %s", torch.__version__)
