def test_filehandler_format(self, temporary_log_directory: local, patch_logging_formatter: MagicMock, patch_logging_filehandler: MagicMock, test_input_name: str, test_input_filename: str) -> None: """Test the file handler is set with the correct logging format.""" # Create a temporary log file, and get its path, if `test_input_filename` is not None if test_input_filename: temporary_log_file = temporary_log_directory.join( test_input_filename) temporary_log_file_path = os.path.join(temporary_log_file.dirname, temporary_log_file.basename) else: temporary_log_file_path = None # Run the `create_logger` function _ = create_logger(test_input_name, temporary_log_file_path) # If a filename is given, check the file handler is set with the correct logging format. Otherwise check that # the file handler is not called if test_input_filename: patch_logging_filehandler.return_value.setFormatter.assert_called_once_with( patch_logging_formatter.return_value) else: assert not patch_logging_filehandler.called
def example_log_file(temporary_log_directory) -> Dict[str, Union[logging.Logger, logging.RootLogger, str]]: """Create a temporary log file for testing purposes.""" # Create a temporary directory to store the log file, and define a file path to a log file temporary_log_file = temporary_log_directory.join("temporary.log") temporary_log_filepath = os.path.join(temporary_log_file.dirname, temporary_log_file.basename) # Create a log file temporary_log = create_logger("temporary_log", temporary_log_filepath) # Return a dictionary containing the log, and its filepath return {"logger": temporary_log, "path": temporary_log_filepath}
def __init__(self, model, data, args): super(Trainer, self).__init__() self.model = model self.data = data['dataloader'] self.optimizer = get_optimizer(self.model.parameters(), args.optim) self._best_valid_loss = float('inf') self.dump_path = args.dump_path self.world_size = args.world_size self.args = args self.logger = create_logger(os.path.join(self.dump_path, 'train.log'), self.args.local_rank) if args.reload_path != "": self.load_checkpoint(args.reload_path)
def test_log_name(self, temporary_log_directory: local, patch_logging_getlogger: MagicMock, test_input_name: str, test_input_filename: str) -> None: """Test the function is assigned the correct name.""" # Create a temporary log file, and get its path, if `test_input_filename` is not None if test_input_filename: temporary_log_file = temporary_log_directory.join( test_input_filename) temporary_log_file_path = os.path.join(temporary_log_file.dirname, temporary_log_file.basename) else: temporary_log_file_path = None # Run the `create_logger` function _ = create_logger(test_input_name, temporary_log_file_path) # Assert the correct name is used patch_logging_getlogger.assert_called_with(test_input_name)
def test_log_format(self, temporary_log_directory: local, patch_logging_formatter: MagicMock, test_input_name: str, test_input_filename: str) -> None: """Test the format of the log.""" # Create a temporary log file, and get its path, if `test_input_filename` is not None if test_input_filename: temporary_log_file = temporary_log_directory.join( test_input_filename) temporary_log_file_path = os.path.join(temporary_log_file.dirname, temporary_log_file.basename) else: temporary_log_file_path = None # Run the `create_logger` function _ = create_logger(test_input_name, temporary_log_file_path) # Assert the correct logging format is applied for the log patch_logging_formatter.assert_called_once_with(EXPECTED_LOG_FORMAT)
def test_log_level(self, temporary_log_directory: local, patch_logging_getlogger: MagicMock, test_input_name: str, test_input_filename: str) -> None: """Test the correct logging level is set.""" # Create a temporary log file, and get its path, if `test_input_filename` is not None if test_input_filename: temporary_log_file = temporary_log_directory.join( test_input_filename) temporary_log_file_path = os.path.join(temporary_log_file.dirname, temporary_log_file.basename) else: temporary_log_file_path = None # Run the `create_logger` function _ = create_logger(test_input_name, temporary_log_file_path) # Assert the correct logging level is set for the log patch_logging_getlogger.return_value.setLevel.assert_called_once_with( logging.DEBUG)
def test_log_output(self, temporary_log_directory: local, patch_logging_getlogger: MagicMock, test_input_name: str, test_input_filename: str) -> None: """Test the function outputs the expected log.""" # Create a temporary log file, and get its path, if `test_input_filename` is not None if test_input_filename: temporary_log_file = temporary_log_directory.join( test_input_filename) temporary_log_file_path = os.path.join(temporary_log_file.dirname, temporary_log_file.basename) else: temporary_log_file_path = None # Run the `create_logger` function test_output = create_logger(test_input_name, temporary_log_file_path) # Assert the output is as expected assert test_output == patch_logging_getlogger.return_value
def test_streamhandler_format(self, temporary_log_directory: local, patch_logging_formatter: MagicMock, patch_logging_streamhandler: MagicMock, test_input_name: str, test_input_filename: str) -> None: """Test the correct log format is used for the stream handler.""" # Create a temporary log file, and get its path, if `test_input_filename` is not None if test_input_filename: temporary_log_file = temporary_log_directory.join( test_input_filename) temporary_log_file_path = os.path.join(temporary_log_file.dirname, temporary_log_file.basename) else: temporary_log_file_path = None # Run the `create_logger` function _ = create_logger(test_input_name, temporary_log_file_path) # Assert the correct log format is set for the stream handler patch_logging_streamhandler.return_value.setFormatter.assert_called_once_with( patch_logging_formatter.return_value)
def test_add_handlers_to_log(self, mocker, temporary_log_directory: local, patch_logging_getlogger: MagicMock, patch_logging_streamhandler: MagicMock, patch_logging_filehandler: MagicMock, test_input_name: str, test_input_filename: str) -> None: """Test the stream handler is added to the log, as well as the file handler, if a filename is given.""" # Create a temporary log file, and get its path, if `test_input_filename` is not None if test_input_filename: temporary_log_file = temporary_log_directory.join( test_input_filename) temporary_log_file_path = os.path.join(temporary_log_file.dirname, temporary_log_file.basename) else: temporary_log_file_path = None # Run the `create_logger` function _ = create_logger(test_input_name, temporary_log_file_path) # If a filename is given, check that the last two handlers added to the log are stream and file handler. # Otherwise, check that only the stream handler is added if test_input_filename: # Define the last two expected calls as the stream and file handlers (in order) test_expected = [ mocker.call(patch_logging_streamhandler.return_value), mocker.call(patch_logging_filehandler.return_value) ] # Assert that the last two calls to `addHandler` are correct assert patch_logging_getlogger.return_value.addHandler.call_args_list[ -2:] == test_expected else: patch_logging_getlogger.return_value.addHandler.assert_called_with( patch_logging_streamhandler.return_value)
from home.src import draw from src.utils import tools, logger from src.utils import database_mysql as db log = logger.create_logger(__name__) ######################################################################################## ### THIS FILE IS FOR REFERENCE / MARKING ONLY (VERSION 1 MOSQUITTO BROKER MYSQL) ####### ##################### THIS FILE IS NOT NEEDED ANYMORE ################################# ######################################################################################## def submit(minTS, maxTS, pub, sub, topic, interval): if interval is not None: interval = int(interval) minTS = tools.strToDT(minTS) maxTS = tools.strToDT(maxTS) script, div = draw.drawGrid(minTS, maxTS, pub, sub, topic, interval) return script, div def generateAllCubes(minTS=None, maxTS=None): conn = db.DBConnect() cursor = conn.cursor() l = list() cursor.execute(db.CNT_QUERY, (minTS, maxTS)) for (prodID, consID, topic, cnt) in cursor: # create a dict
except Exception as e: self.logger.debug("Exception in training loop") self.logger.debug(e.message) def test(self, n_tests): self.transformer.eval() lang = 0 get_iterator = self.get_lm_iterator(lang=lang, train=True, add_noise=True) train_iterator = get_iterator() for i in range(n_tests): batch_dict = next(train_iterator) #self.greedy_decoding(batch_dict, lang) self.output_samples(batch_dict, lang, lang) if __name__ == "__main__": logger = create_logger("logs/en_language_model.log") parser = get_parser() data_params = parser.parse_args() check_all_data_params(data_params) model = Transformer(data_params=data_params, logger=logger, init_emb=False) trainer = LanguageModeling(model) trainer.train(3000) trainer.save_model("en_language_model.pth") logger.info("testing trained model") trainer.test(10) logger.info("testing loaded model") trainer.load_model("en_language_model.pth") trainer.test(10)
self.transformer.eval() lang1 = 0 lang2 = 1 get_iterator = self.get_para_iterator(lang1=lang1, lang2=lang2, train=False, add_noise=False) train_iterator = get_iterator() for i in range(n_tests): batch_dict = next(train_iterator) #self.greedy_decoding(batch_dict, lang1, lang2) self.output_samples(batch_dict, lang1, lang2) loss = self.translation_loss(batch_dict, lang1, lang2) self.logger.info("translation loss", loss) if __name__ == "__main__": logger = create_logger("logs/para_trainer.log") parser = get_parser() data_params = parser.parse_args() check_all_data_params(data_params) model = Transformer(data_params=data_params, logger=logger, init_emb=True, embd_file="corpora/mono/all.en-fr.60000.vec") trainer = ParallelTrainer(model) # test iterator # get_iter = trainer.get_para_iterator(lang1=0, lang2=1, train=False, add_noise=False) # iter = get_iter() # batch_dict = next(iter) # prev_output = batch_dict["prev_output"] # tgt_mask = batch_dict["tgt_mask"]