def test_clock(): clock = Clock() clock.start() for _ in range(100): pass clock.end(wall=True) clock.end(wall=False) print(clock.process_time) print(clock.wall_time) store_dict = {} with Clock(store_dict, config_logging(logger="clock")): for _ in range(100): pass assert "wall_time" in store_dict assert "process_time" in store_dict with Clock(store_dict, config_logging(logger="clock"), tips="testing clock"): for _ in range(100): pass
def numerical_check(_net, _cfg: Configuration, train_data, test_data, dump_result=False): # pragma: no cover ctx = _cfg.ctx batch_size = _cfg.batch_size _net.initialize(ctx=ctx) bp_loss_f = get_bp_loss(**_cfg.loss_params) loss_function = {} loss_function.update(bp_loss_f) from longling.ML.MxnetHelper.glue import module from longling.ML.toolkit import EvalFormatter as Formatter from longling.ML.toolkit import MovingLoss from tqdm import tqdm loss_monitor = MovingLoss(loss_function) progress_monitor = tqdm if dump_result: from longling import config_logging validation_logger = config_logging( filename=path_append(_cfg.model_dir, "result.log"), logger="%s-validation" % _cfg.model_name, mode="w", log_format="%(message)s", ) evaluation_formatter = Formatter( logger=validation_logger, dump_file=_cfg.validation_result_file, ) else: evaluation_formatter = Formatter() # train check trainer = module.Module.get_trainer( _net, optimizer=_cfg.optimizer, optimizer_params=_cfg.optimizer_params, select=_cfg.train_select ) for epoch in range(_cfg.begin_epoch, _cfg.end_epoch): for batch_data in progress_monitor(train_data, "Epoch: %s" % epoch): fit_f( net=_net, batch_size=batch_size, batch_data=batch_data, trainer=trainer, bp_loss_f=bp_loss_f, loss_function=loss_function, loss_monitor=loss_monitor, ctx=ctx, ) if epoch % 1 == 0: if epoch % 1 == 0: print( evaluation_formatter( epoch=epoch, loss_name_value=dict(loss_monitor.items()), eval_name_value=eval_f(_net, test_data, ctx=ctx), extra_info=None, dump=True, )[0] )
def test_root_logger(): set_logging_info() logging.info("test_log") logger = config_logging(level="info", console_log_level="info", datefmt="default", enable_colored=True) logger.info("test_log")
def test_json_log(log_root): json_log_path = path_append(log_root, "test_json_log.json", to_str=True) json_logger = config_logging( filename=json_log_path, file_format="%(message)s", logger="test.json", mode="w", ) json_logger.info(json.dumps({"name": "test"})) with open(json_log_path) as f: assert "name" in json.loads(f.readline())
def __init__(self, logger=None, log_rf=None, *args, **kwargs): # components self.tester = None self.reward = None self._logger = logger if logger is not None else config_logging( logger="Env", console_log_level="info") if log_rf is None: self._log_rf = config_logging(logger="Env_RF", console_log_level="info", log_format=json.dumps("%(message)s")) elif isinstance(log_rf, str): self._log_rf = config_logging( filename=log_rf, logger="Env_RF", file_format=json.dumps("%(message)s")) else: self._log_rf = log_rf # episode relevant self._learner = None self._path = None
def test_env(tmp_path): log_rf = path_append(tmp_path, "kss_rf.json", to_str=True) Env(log_rf=log_rf) Env() _env = Env(log_rf=config_logging(logger="test_env")) _env.render() _env.reset() assert set(_env.summary_episode()) == set(return_key_env_summary_episode) assert True
def __init__(self, logger=None, *args, **kwargs): # components self._logger = logger if logger is not None else config_logging( logger="Agent", console_log_level="info") # following variables is episode variables # the learning path composed by learning items is generated by the `step` function self._path = None # interactions refers to the observation sequence self._interactions = None self._learner = None self._learner_learning_history = None self._learner_exercise_history = None self._episode_reward = None
def file_logger(log_path): return config_logging(filename=log_path, logger="test.file")
# coding: utf-8 # 2020/5/8 @ tongshiwei from longling import config_logging from EduSim.Envs.KSS import KSSEnv, kss_train_eval, KSSAgent env = KSSEnv() kss_train_eval(KSSAgent(env.action_num), env, 20, 4000, logger=config_logging(logger="kss", console_log_level="debug", level="debug"), level="summary", board_dir="./kss_logs")
def train(net, cfg, loss_function, trainer, train_data, test_data=None, params_save=False, dump_result=False, progress_monitor=None, *, fit_f, eval_f=None, net_init=None, get_net=None, get_loss=None, get_trainer=None, save_params=None, enable_hyper_search=False, reporthook=None, final_reporthook=None, primary_key=None, eval_epoch=1, loss_dict2tmt_loss=None, epoch_lr_scheduler=None, batch_lr_scheduler=None, loss_as_dict=False, verbose=None, dump_cfg=None, **cfg_kwargs): if enable_hyper_search: assert get_net is not None cfg_kwargs, reporthook, final_reporthook, tag = prepare_hyper_search( cfg_kwargs, reporthook, final_reporthook, primary_key=primary_key, with_keys="Epoch", dump=params_save) dump_result = tag verbose = tag if verbose is None else verbose cfg.update(**cfg_kwargs) print("hyper search enabled") print(cfg) verbose = True if verbose is None else verbose dump_cfg = dump_cfg if dump_cfg is not None else params_save if dump_cfg: cfg.dump() net = net if get_net is None else get_net(**cfg.hyper_params) if net_init is not None: net_init(net, cfg=cfg, initializer_kwargs=cfg.init_params) train_ctx = cfg.ctx if cfg.train_ctx is None else cfg.train_ctx eval_ctx = cfg.ctx if cfg.eval_ctx is None else cfg.eval_ctx batch_size = cfg.batch_size loss_function = get_loss( **cfg.loss_params) if get_loss is not None else loss_function if isinstance(loss_function, dict): _loss_function = loss_function else: if hasattr(loss_function, "__name__"): loss_name = loss_function.__name__ elif hasattr(loss_function, "__class__"): loss_name = loss_function.__class__.__name__ else: # pragma: no cover loss_name = "loss" loss_function = {loss_name: loss_function} if loss_dict2tmt_loss is not None: loss_function = loss_dict2tmt_loss(loss_function) _loss_function = list(loss_function.values() )[0] if loss_as_dict is False else loss_function loss_monitor = MovingLoss(loss_function) if progress_monitor is None and loss_dict2tmt_loss is not None: progress_monitor = ConsoleProgressMonitor( indexes={"Loss": [name for name in loss_function]}, values={"Loss": loss_monitor.losses}, player_type="epoch", total_epoch=cfg.end_epoch - 1, silent=not verbose) elif progress_monitor is None or progress_monitor == "tqdm": def progress_monitor(x, e): return tqdm(x, "Epoch: %s" % e, disable=not verbose) if dump_result: from longling import config_logging validation_logger = config_logging( filename=path_append(cfg.model_dir, cfg.get("result_log", RESULT_LOG)), logger="%s-validation" % cfg.model_name, mode="w", log_format="%(message)s", ) evaluation_formatter = Formatter( logger=validation_logger, dump_file=cfg.validation_result_file, ) else: evaluation_formatter = Formatter() # train check if get_trainer is not None: trainer = get_trainer(net, optimizer=cfg.optimizer, optimizer_params=cfg.optimizer_params, select=cfg.train_select, lr_params=cfg.lr_params) if batch_lr_scheduler is True: trainer, batch_lr_scheduler = trainer elif epoch_lr_scheduler is True: trainer, epoch_lr_scheduler = trainer for epoch in range(cfg.begin_epoch, cfg.end_epoch): for i, batch_data in enumerate(progress_monitor(train_data, epoch)): fit_f( net, batch_size=batch_size, batch_data=batch_data, trainer=trainer, loss_function=_loss_function, loss_monitor=loss_monitor, ctx=train_ctx, ) if batch_lr_scheduler is not None: batch_lr_scheduler.step() if cfg.lr_params and "update_params" in cfg.lr_params and cfg.end_epoch - cfg.begin_epoch - 1 > 0: cfg.logger.info("reset trainer") lr_params = cfg.lr_params.pop("update_params") lr_update_params = dict(batches_per_epoch=i + 1, lr=cfg.optimizer_params["learning_rate"], update_epoch=lr_params.get( "update_epoch", cfg.end_epoch - cfg.begin_epoch - 1)) lr_update_params.update(lr_params) assert get_trainer is not None trainer = get_trainer(net, optimizer=cfg.optimizer, optimizer_params=cfg.optimizer_params, lr_params=lr_update_params, select=cfg.train_select, logger=cfg.logger) if test_data is not None and epoch % eval_epoch == 0: msg, data = evaluation_formatter( iteration=epoch, loss_name_value=dict(loss_monitor.items()), eval_name_value=eval_f(net, test_data, ctx=eval_ctx, verbose=verbose, **cfg.get("eval_params", {})), extra_info=None, dump=dump_result, keep={"msg", "data"}) print(msg) if reporthook is not None: reporthook(data) # optional loss_monitor.reset() if params_save and (epoch % cfg.save_epoch == 0 or epoch == cfg.end_epoch - 1): assert save_params is not None params_path = get_epoch_params_filepath(cfg.model_name, epoch, cfg.model_dir) cfg.logger.info("save model params to %s, with select='%s'" % (params_path, cfg.save_select)) save_params(params_path, net, select=cfg.save_select) if epoch_lr_scheduler is not None: epoch_lr_scheduler.step() if final_reporthook is not None: final_reporthook()
# coding: utf-8 # 2020/1/3 @ tongshiwei from longling import config_logging config_logging(logger="CangJie", console_log_level="info") # These constants are from gluonnlp UNK_TOKEN = '<unk>' BOS_TOKEN = '<bos>' EOS_TOKEN = '<eos>' PAD_TOKEN = '<pad>'
def numerical_check(_net, _cfg: Configuration, train_data, test_data, dump_result=False, reporthook=None, final_reporthook=None): # pragma: no cover ctx = _cfg.ctx _net = set_device(_net, ctx) bp_loss_f = get_bp_loss(ctx, **_cfg.loss_params) loss_function = {} loss_function.update(bp_loss_f) from longling.ML.toolkit import EpochEvalFMT as Formatter from longling.ML.toolkit import MovingLoss from tqdm import tqdm loss_monitor = MovingLoss(loss_function) progress_monitor = tqdm if dump_result: from longling import config_logging validation_logger = config_logging( filename=path_append(_cfg.model_dir, "result.log"), logger="%s-validation" % _cfg.model_name, mode="w", log_format="%(message)s", ) evaluation_formatter = Formatter( logger=validation_logger, dump_file=_cfg.validation_result_file, ) else: evaluation_formatter = Formatter() # train check from longling.ML.PytorchHelper.toolkit.optimizer import get_trainer trainer = get_trainer( _net, optimizer=_cfg.optimizer, optimizer_params=_cfg.optimizer_params, select=_cfg.train_select ) for epoch in range(_cfg.begin_epoch, _cfg.end_epoch): for batch_data in progress_monitor(train_data, "Epoch: %s" % epoch): fit_f( net=_net, batch_data=batch_data, trainer=trainer, bp_loss_f=bp_loss_f, loss_function=loss_function, loss_monitor=loss_monitor, ) if epoch % 1 == 0: msg, data = evaluation_formatter( epoch=epoch, loss_name_value=dict(loss_monitor.items()), eval_name_value=eval_f(_net, test_data, ctx=ctx), extra_info=None, dump=dump_result, ) print(msg) if reporthook is not None: reporthook(data) # optional, whether reset the loss at the end of each epoch loss_monitor.reset() if final_reporthook is not None: final_reporthook()
def __init__(self, logger=None): self._initial_score = None self._final_score = None self._learner = None self._logger = logger if logger else config_logging(logger="Tester")
# coding: utf-8 # create by tongshiwei on 2019/7/2 import codecs import csv import json import networkx as nx import pandas from longling import wf_open, config_logging from tqdm import tqdm logger = config_logging(logger="junyi", console_log_level="info") def build_ku_dict(source, target): with codecs.open(source, encoding="utf-8") as f, wf_open(target) as wf: f.readline() idx = 0 vertex_dict = {} for line in tqdm(csv.reader(f)): if line[0] not in vertex_dict: vertex_dict[line[0]] = idx idx += 1 logger.info("vertex num: %s" % len(vertex_dict)) json.dump(vertex_dict, wf, indent=2) def extract_prerequisite(source, target, ku_dict): """in target: (A, B) means predecessor --> successor""" with codecs.open(
# coding: utf-8 from longling import config_logging logger = config_logging(logger="spider", console_log_level="info")
def logger(): return config_logging(logger="test")
# coding: utf-8 # 2020/4/16 @ tongshiwei import functools from pathlib import PurePath import json import math import numpy as np from longling import loading as _loading, as_out_io, PATH_TYPE, json_load, close_io from longling import AddPrinter, type_from_name, config_logging, as_list from sklearn.utils.validation import check_random_state from sklearn.model_selection import KFold from longling import concurrent_pool _logger = config_logging(logger="dataset", console_log_level="info") loading = functools.partial(_loading, src_type="text") def _get_s_points(length: int, ratio: list) -> list: """ Returns ------- s_points: list of int Examples -------- >>> _get_s_points(10, [0.1, 0.9]) [0, 1, 10] """ cum_ratios = np.cumsum([0.0] + ratio) s_points = [math.ceil(length * ratio) for ratio in cum_ratios]
# create by tongshiwei on 2019/7/2 __all__ = ["url_dict", "get_data"] import os from urllib.request import urlretrieve import requests from bs4 import BeautifulSoup from longling import config_logging, LogLevel, path_append # from longling.spider import download_data from .utils import decompress DEFAULT_DATADIR = path_append("../", "data/", to_str=True) config_logging(logger="downloader", console_log_level=LogLevel.INFO) prefix = 'http://base.ustc.edu.cn/data/' url_dict = { "assistment-2009-2010-skill": "http://base.ustc.edu.cn/data/ASSISTment/2009_skill_builder_data_corrected.zip", "assistment-2012-2013-non-skill": "http://base.ustc.edu.cn/data/ASSISTment/2012-2013-data-with-predictions-4-final.zip", "assistment-2015": "http://base.ustc.edu.cn/data/ASSISTment/2015_100_skill_builders_main_problems.zip", "junyi": "http://base.ustc.edu.cn/data/JunyiAcademy_Math_Practicing_Log/junyi.rar", "KDD-CUP-2010": "http://base.ustc.edu.cn/data/KDD_Cup_2010/", "slepemapy.cz": "http://base.ustc.edu.cn/data/slepemapy.cz/" }
def json_logger(json_log_path): return config_logging(filename=json_log_path, file_format="%(message)s", logger="test.json")
def numerical_check(_net, _cfg: Configuration, train_data, test_data, dump_result=False, reporthook=None, final_reporthook=None): # pragma: no cover ctx = _cfg.ctx batch_size = _cfg.batch_size _net.initialize(ctx=ctx) bp_loss_f = get_bp_loss(**_cfg.loss_params) loss_function = {} loss_function.update(bp_loss_f) from longling.ML.MxnetHelper.glue import module from longling.ML.toolkit import EpochEvalFMT as Formatter from longling.ML.toolkit import MovingLoss from tqdm import tqdm loss_monitor = MovingLoss(loss_function) progress_monitor = tqdm if dump_result: from longling import config_logging validation_logger = config_logging( filename=path_append(_cfg.model_dir, "result.log"), logger="%s-validation" % _cfg.model_name, mode="w", log_format="%(message)s", ) evaluation_formatter = Formatter( logger=validation_logger, dump_file=_cfg.validation_result_file, ) else: evaluation_formatter = Formatter() # train check trainer = module.Module.get_trainer(_net, optimizer=_cfg.optimizer, optimizer_params=_cfg.optimizer_params, select=_cfg.train_select) for epoch in range(_cfg.begin_epoch, _cfg.end_epoch): for i, batch_data in enumerate( progress_monitor(train_data, "Epoch: %s" % epoch)): fit_f( net=_net, batch_size=batch_size, batch_data=batch_data, trainer=trainer, bp_loss_f=bp_loss_f, loss_function=loss_function, loss_monitor=loss_monitor, ctx=ctx, ) if _cfg.lr_params and "update_params" in _cfg.lr_params: _cfg.logger.info("reset trainer") lr_params = _cfg.lr_params.pop("update_params") lr_update_params = dict(batches_per_epoch=i + 1, lr=_cfg.optimizer_params["learning_rate"], update_epoch=lr_params.get( "update_epoch", _cfg.end_epoch - _cfg.begin_epoch - 1)) lr_update_params.update(lr_params) trainer = module.Module.get_trainer( _net, optimizer=_cfg.optimizer, optimizer_params=_cfg.optimizer_params, lr_params=lr_update_params, select=_cfg.train_select, logger=_cfg.logger) if epoch % 1 == 0: msg, data = evaluation_formatter(iteration=epoch, loss_name_value=dict( loss_monitor.items()), eval_name_value=eval_f(_net, test_data, ctx=ctx), extra_info=None, dump=dump_result, keep={"msg", "data"}) print(msg) if reporthook is not None: reporthook(data) if final_reporthook is not None: final_reporthook()
# coding: utf-8 # Copyright @tongshiwei from longling import config_logging config_logging(logger="RFAG", console_log_level="info") from .RFAG import RFAG __all__ = ["RFAG"]