def test_filehandler_format(self, temporary_log_directory: local,
patch_logging_formatter: MagicMock,
patch_logging_filehandler: MagicMock,
test_input_name: str,
test_input_filename: str) -> None:
"""Test the file handler is set with the correct logging format."""
# Create a temporary log file, and get its path, if `test_input_filename` is not None
if test_input_filename:
temporary_log_file = temporary_log_directory.join(
test_input_filename)
temporary_log_file_path = os.path.join(temporary_log_file.dirname,
temporary_log_file.basename)
else:
temporary_log_file_path = None
# Run the `create_logger` function
_ = create_logger(test_input_name, temporary_log_file_path)
# If a filename is given, check the file handler is set with the correct logging format. Otherwise check that
# the file handler is not called
if test_input_filename:
patch_logging_filehandler.return_value.setFormatter.assert_called_once_with(
patch_logging_formatter.return_value)
else:
assert not patch_logging_filehandler.called
def example_log_file(temporary_log_directory) -> Dict[str, Union[logging.Logger, logging.RootLogger, str]]:
"""Create a temporary log file for testing purposes."""
# Create a temporary directory to store the log file, and define a file path to a log file
temporary_log_file = temporary_log_directory.join("temporary.log")
temporary_log_filepath = os.path.join(temporary_log_file.dirname, temporary_log_file.basename)
# Create a log file
temporary_log = create_logger("temporary_log", temporary_log_filepath)
# Return a dictionary containing the log, and its filepath
return {"logger": temporary_log, "path": temporary_log_filepath}
def __init__(self, model, data, args):
super(Trainer, self).__init__()
self.model = model
self.data = data['dataloader']
self.optimizer = get_optimizer(self.model.parameters(), args.optim)
self._best_valid_loss = float('inf')
self.dump_path = args.dump_path
self.world_size = args.world_size
self.args = args
self.logger = create_logger(os.path.join(self.dump_path, 'train.log'),
self.args.local_rank)
if args.reload_path != "":
self.load_checkpoint(args.reload_path)
def test_log_name(self, temporary_log_directory: local,
patch_logging_getlogger: MagicMock, test_input_name: str,
test_input_filename: str) -> None:
"""Test the function is assigned the correct name."""
# Create a temporary log file, and get its path, if `test_input_filename` is not None
if test_input_filename:
temporary_log_file = temporary_log_directory.join(
test_input_filename)
temporary_log_file_path = os.path.join(temporary_log_file.dirname,
temporary_log_file.basename)
else:
temporary_log_file_path = None
# Run the `create_logger` function
_ = create_logger(test_input_name, temporary_log_file_path)
# Assert the correct name is used
patch_logging_getlogger.assert_called_with(test_input_name)
def test_log_format(self, temporary_log_directory: local,
patch_logging_formatter: MagicMock,
test_input_name: str,
test_input_filename: str) -> None:
"""Test the format of the log."""
# Create a temporary log file, and get its path, if `test_input_filename` is not None
if test_input_filename:
temporary_log_file = temporary_log_directory.join(
test_input_filename)
temporary_log_file_path = os.path.join(temporary_log_file.dirname,
temporary_log_file.basename)
else:
temporary_log_file_path = None
# Run the `create_logger` function
_ = create_logger(test_input_name, temporary_log_file_path)
# Assert the correct logging format is applied for the log
patch_logging_formatter.assert_called_once_with(EXPECTED_LOG_FORMAT)
def test_log_level(self, temporary_log_directory: local,
patch_logging_getlogger: MagicMock,
test_input_name: str, test_input_filename: str) -> None:
"""Test the correct logging level is set."""
# Create a temporary log file, and get its path, if `test_input_filename` is not None
if test_input_filename:
temporary_log_file = temporary_log_directory.join(
test_input_filename)
temporary_log_file_path = os.path.join(temporary_log_file.dirname,
temporary_log_file.basename)
else:
temporary_log_file_path = None
# Run the `create_logger` function
_ = create_logger(test_input_name, temporary_log_file_path)
# Assert the correct logging level is set for the log
patch_logging_getlogger.return_value.setLevel.assert_called_once_with(
logging.DEBUG)
def test_log_output(self, temporary_log_directory: local,
patch_logging_getlogger: MagicMock,
test_input_name: str,
test_input_filename: str) -> None:
"""Test the function outputs the expected log."""
# Create a temporary log file, and get its path, if `test_input_filename` is not None
if test_input_filename:
temporary_log_file = temporary_log_directory.join(
test_input_filename)
temporary_log_file_path = os.path.join(temporary_log_file.dirname,
temporary_log_file.basename)
else:
temporary_log_file_path = None
# Run the `create_logger` function
test_output = create_logger(test_input_name, temporary_log_file_path)
# Assert the output is as expected
assert test_output == patch_logging_getlogger.return_value
def test_streamhandler_format(self, temporary_log_directory: local,
patch_logging_formatter: MagicMock,
patch_logging_streamhandler: MagicMock,
test_input_name: str,
test_input_filename: str) -> None:
"""Test the correct log format is used for the stream handler."""
# Create a temporary log file, and get its path, if `test_input_filename` is not None
if test_input_filename:
temporary_log_file = temporary_log_directory.join(
test_input_filename)
temporary_log_file_path = os.path.join(temporary_log_file.dirname,
temporary_log_file.basename)
else:
temporary_log_file_path = None
# Run the `create_logger` function
_ = create_logger(test_input_name, temporary_log_file_path)
# Assert the correct log format is set for the stream handler
patch_logging_streamhandler.return_value.setFormatter.assert_called_once_with(
patch_logging_formatter.return_value)
def test_add_handlers_to_log(self, mocker, temporary_log_directory: local,
patch_logging_getlogger: MagicMock,
patch_logging_streamhandler: MagicMock,
patch_logging_filehandler: MagicMock,
test_input_name: str,
test_input_filename: str) -> None:
"""Test the stream handler is added to the log, as well as the file handler, if a filename is given."""
# Create a temporary log file, and get its path, if `test_input_filename` is not None
if test_input_filename:
temporary_log_file = temporary_log_directory.join(
test_input_filename)
temporary_log_file_path = os.path.join(temporary_log_file.dirname,
temporary_log_file.basename)
else:
temporary_log_file_path = None
# Run the `create_logger` function
_ = create_logger(test_input_name, temporary_log_file_path)
# If a filename is given, check that the last two handlers added to the log are stream and file handler.
# Otherwise, check that only the stream handler is added
if test_input_filename:
# Define the last two expected calls as the stream and file handlers (in order)
test_expected = [
mocker.call(patch_logging_streamhandler.return_value),
mocker.call(patch_logging_filehandler.return_value)
]
# Assert that the last two calls to `addHandler` are correct
assert patch_logging_getlogger.return_value.addHandler.call_args_list[
-2:] == test_expected
else:
patch_logging_getlogger.return_value.addHandler.assert_called_with(
patch_logging_streamhandler.return_value)
from home.src import draw
from src.utils import tools, logger
from src.utils import database_mysql as db
log = logger.create_logger(__name__)
########################################################################################
### THIS FILE IS FOR REFERENCE / MARKING ONLY (VERSION 1 MOSQUITTO BROKER MYSQL) #######
##################### THIS FILE IS NOT NEEDED ANYMORE #################################
########################################################################################
def submit(minTS, maxTS, pub, sub, topic, interval):
if interval is not None:
interval = int(interval)
minTS = tools.strToDT(minTS)
maxTS = tools.strToDT(maxTS)
script, div = draw.drawGrid(minTS, maxTS, pub, sub, topic, interval)
return script, div
def generateAllCubes(minTS=None, maxTS=None):
conn = db.DBConnect()
cursor = conn.cursor()
l = list()
cursor.execute(db.CNT_QUERY, (minTS, maxTS))
for (prodID, consID, topic, cnt) in cursor:
# create a dict
except Exception as e:
self.logger.debug("Exception in training loop")
self.logger.debug(e.message)
def test(self, n_tests):
self.transformer.eval()
lang = 0
get_iterator = self.get_lm_iterator(lang=lang, train=True, add_noise=True)
train_iterator = get_iterator()
for i in range(n_tests):
batch_dict = next(train_iterator)
#self.greedy_decoding(batch_dict, lang)
self.output_samples(batch_dict, lang, lang)
if __name__ == "__main__":
logger = create_logger("logs/en_language_model.log")
parser = get_parser()
data_params = parser.parse_args()
check_all_data_params(data_params)
model = Transformer(data_params=data_params, logger=logger, init_emb=False)
trainer = LanguageModeling(model)
trainer.train(3000)
trainer.save_model("en_language_model.pth")
logger.info("testing trained model")
trainer.test(10)
logger.info("testing loaded model")
trainer.load_model("en_language_model.pth")
trainer.test(10)
self.transformer.eval()
lang1 = 0
lang2 = 1
get_iterator = self.get_para_iterator(lang1=lang1, lang2=lang2, train=False, add_noise=False)
train_iterator = get_iterator()
for i in range(n_tests):
batch_dict = next(train_iterator)
#self.greedy_decoding(batch_dict, lang1, lang2)
self.output_samples(batch_dict, lang1, lang2)
loss = self.translation_loss(batch_dict, lang1, lang2)
self.logger.info("translation loss", loss)
if __name__ == "__main__":
logger = create_logger("logs/para_trainer.log")
parser = get_parser()
data_params = parser.parse_args()
check_all_data_params(data_params)
model = Transformer(data_params=data_params, logger=logger,
init_emb=True, embd_file="corpora/mono/all.en-fr.60000.vec")
trainer = ParallelTrainer(model)
# test iterator
# get_iter = trainer.get_para_iterator(lang1=0, lang2=1, train=False, add_noise=False)
# iter = get_iter()
# batch_dict = next(iter)
# prev_output = batch_dict["prev_output"]
# tgt_mask = batch_dict["tgt_mask"]