def test_clock():
clock = Clock()
clock.start()
for _ in range(100):
pass
clock.end(wall=True)
clock.end(wall=False)
print(clock.process_time)
print(clock.wall_time)
store_dict = {}
with Clock(store_dict, config_logging(logger="clock")):
for _ in range(100):
pass
assert "wall_time" in store_dict
assert "process_time" in store_dict
with Clock(store_dict,
config_logging(logger="clock"),
tips="testing clock"):
for _ in range(100):
pass
def numerical_check(_net, _cfg: Configuration, train_data, test_data, dump_result=False): # pragma: no cover
ctx = _cfg.ctx
batch_size = _cfg.batch_size
_net.initialize(ctx=ctx)
bp_loss_f = get_bp_loss(**_cfg.loss_params)
loss_function = {}
loss_function.update(bp_loss_f)
from longling.ML.MxnetHelper.glue import module
from longling.ML.toolkit import EvalFormatter as Formatter
from longling.ML.toolkit import MovingLoss
from tqdm import tqdm
loss_monitor = MovingLoss(loss_function)
progress_monitor = tqdm
if dump_result:
from longling import config_logging
validation_logger = config_logging(
filename=path_append(_cfg.model_dir, "result.log"),
logger="%s-validation" % _cfg.model_name,
mode="w",
log_format="%(message)s",
)
evaluation_formatter = Formatter(
logger=validation_logger,
dump_file=_cfg.validation_result_file,
)
else:
evaluation_formatter = Formatter()
# train check
trainer = module.Module.get_trainer(
_net, optimizer=_cfg.optimizer,
optimizer_params=_cfg.optimizer_params,
select=_cfg.train_select
)
for epoch in range(_cfg.begin_epoch, _cfg.end_epoch):
for batch_data in progress_monitor(train_data, "Epoch: %s" % epoch):
fit_f(
net=_net, batch_size=batch_size, batch_data=batch_data,
trainer=trainer, bp_loss_f=bp_loss_f,
loss_function=loss_function,
loss_monitor=loss_monitor,
ctx=ctx,
)
if epoch % 1 == 0:
if epoch % 1 == 0:
print(
evaluation_formatter(
epoch=epoch,
loss_name_value=dict(loss_monitor.items()),
eval_name_value=eval_f(_net, test_data, ctx=ctx),
extra_info=None,
dump=True,
)[0]
)
def test_root_logger():
set_logging_info()
logging.info("test_log")
logger = config_logging(level="info",
console_log_level="info",
datefmt="default",
enable_colored=True)
logger.info("test_log")
def test_json_log(log_root):
json_log_path = path_append(log_root, "test_json_log.json", to_str=True)
json_logger = config_logging(
filename=json_log_path,
file_format="%(message)s",
logger="test.json",
mode="w",
)
json_logger.info(json.dumps({"name": "test"}))
with open(json_log_path) as f:
assert "name" in json.loads(f.readline())
def __init__(self, logger=None, log_rf=None, *args, **kwargs):
# components
self.tester = None
self.reward = None
self._logger = logger if logger is not None else config_logging(
logger="Env", console_log_level="info")
if log_rf is None:
self._log_rf = config_logging(logger="Env_RF",
console_log_level="info",
log_format=json.dumps("%(message)s"))
elif isinstance(log_rf, str):
self._log_rf = config_logging(
filename=log_rf,
logger="Env_RF",
file_format=json.dumps("%(message)s"))
else:
self._log_rf = log_rf
# episode relevant
self._learner = None
self._path = None
def test_env(tmp_path):
log_rf = path_append(tmp_path, "kss_rf.json", to_str=True)
Env(log_rf=log_rf)
Env()
_env = Env(log_rf=config_logging(logger="test_env"))
_env.render()
_env.reset()
assert set(_env.summary_episode()) == set(return_key_env_summary_episode)
assert True
def __init__(self, logger=None, *args, **kwargs):
# components
self._logger = logger if logger is not None else config_logging(
logger="Agent", console_log_level="info")
# following variables is episode variables
# the learning path composed by learning items is generated by the `step` function
self._path = None
# interactions refers to the observation sequence
self._interactions = None
self._learner = None
self._learner_learning_history = None
self._learner_exercise_history = None
self._episode_reward = None
def file_logger(log_path):
return config_logging(filename=log_path, logger="test.file")
# coding: utf-8
# 2020/5/8 @ tongshiwei
from longling import config_logging
from EduSim.Envs.KSS import KSSEnv, kss_train_eval, KSSAgent
env = KSSEnv()
kss_train_eval(KSSAgent(env.action_num),
env,
20,
4000,
logger=config_logging(logger="kss",
console_log_level="debug",
level="debug"),
level="summary",
board_dir="./kss_logs")
def train(net,
cfg,
loss_function,
trainer,
train_data,
test_data=None,
params_save=False,
dump_result=False,
progress_monitor=None,
*,
fit_f,
eval_f=None,
net_init=None,
get_net=None,
get_loss=None,
get_trainer=None,
save_params=None,
enable_hyper_search=False,
reporthook=None,
final_reporthook=None,
primary_key=None,
eval_epoch=1,
loss_dict2tmt_loss=None,
epoch_lr_scheduler=None,
batch_lr_scheduler=None,
loss_as_dict=False,
verbose=None,
dump_cfg=None,
**cfg_kwargs):
if enable_hyper_search:
assert get_net is not None
cfg_kwargs, reporthook, final_reporthook, tag = prepare_hyper_search(
cfg_kwargs,
reporthook,
final_reporthook,
primary_key=primary_key,
with_keys="Epoch",
dump=params_save)
dump_result = tag
verbose = tag if verbose is None else verbose
cfg.update(**cfg_kwargs)
print("hyper search enabled")
print(cfg)
verbose = True if verbose is None else verbose
dump_cfg = dump_cfg if dump_cfg is not None else params_save
if dump_cfg:
cfg.dump()
net = net if get_net is None else get_net(**cfg.hyper_params)
if net_init is not None:
net_init(net, cfg=cfg, initializer_kwargs=cfg.init_params)
train_ctx = cfg.ctx if cfg.train_ctx is None else cfg.train_ctx
eval_ctx = cfg.ctx if cfg.eval_ctx is None else cfg.eval_ctx
batch_size = cfg.batch_size
loss_function = get_loss(
**cfg.loss_params) if get_loss is not None else loss_function
if isinstance(loss_function, dict):
_loss_function = loss_function
else:
if hasattr(loss_function, "__name__"):
loss_name = loss_function.__name__
elif hasattr(loss_function, "__class__"):
loss_name = loss_function.__class__.__name__
else: # pragma: no cover
loss_name = "loss"
loss_function = {loss_name: loss_function}
if loss_dict2tmt_loss is not None:
loss_function = loss_dict2tmt_loss(loss_function)
_loss_function = list(loss_function.values()
)[0] if loss_as_dict is False else loss_function
loss_monitor = MovingLoss(loss_function)
if progress_monitor is None and loss_dict2tmt_loss is not None:
progress_monitor = ConsoleProgressMonitor(
indexes={"Loss": [name for name in loss_function]},
values={"Loss": loss_monitor.losses},
player_type="epoch",
total_epoch=cfg.end_epoch - 1,
silent=not verbose)
elif progress_monitor is None or progress_monitor == "tqdm":
def progress_monitor(x, e):
return tqdm(x, "Epoch: %s" % e, disable=not verbose)
if dump_result:
from longling import config_logging
validation_logger = config_logging(
filename=path_append(cfg.model_dir,
cfg.get("result_log", RESULT_LOG)),
logger="%s-validation" % cfg.model_name,
mode="w",
log_format="%(message)s",
)
evaluation_formatter = Formatter(
logger=validation_logger,
dump_file=cfg.validation_result_file,
)
else:
evaluation_formatter = Formatter()
# train check
if get_trainer is not None:
trainer = get_trainer(net,
optimizer=cfg.optimizer,
optimizer_params=cfg.optimizer_params,
select=cfg.train_select,
lr_params=cfg.lr_params)
if batch_lr_scheduler is True:
trainer, batch_lr_scheduler = trainer
elif epoch_lr_scheduler is True:
trainer, epoch_lr_scheduler = trainer
for epoch in range(cfg.begin_epoch, cfg.end_epoch):
for i, batch_data in enumerate(progress_monitor(train_data, epoch)):
fit_f(
net,
batch_size=batch_size,
batch_data=batch_data,
trainer=trainer,
loss_function=_loss_function,
loss_monitor=loss_monitor,
ctx=train_ctx,
)
if batch_lr_scheduler is not None:
batch_lr_scheduler.step()
if cfg.lr_params and "update_params" in cfg.lr_params and cfg.end_epoch - cfg.begin_epoch - 1 > 0:
cfg.logger.info("reset trainer")
lr_params = cfg.lr_params.pop("update_params")
lr_update_params = dict(batches_per_epoch=i + 1,
lr=cfg.optimizer_params["learning_rate"],
update_epoch=lr_params.get(
"update_epoch",
cfg.end_epoch - cfg.begin_epoch - 1))
lr_update_params.update(lr_params)
assert get_trainer is not None
trainer = get_trainer(net,
optimizer=cfg.optimizer,
optimizer_params=cfg.optimizer_params,
lr_params=lr_update_params,
select=cfg.train_select,
logger=cfg.logger)
if test_data is not None and epoch % eval_epoch == 0:
msg, data = evaluation_formatter(
iteration=epoch,
loss_name_value=dict(loss_monitor.items()),
eval_name_value=eval_f(net,
test_data,
ctx=eval_ctx,
verbose=verbose,
**cfg.get("eval_params", {})),
extra_info=None,
dump=dump_result,
keep={"msg", "data"})
print(msg)
if reporthook is not None:
reporthook(data)
# optional
loss_monitor.reset()
if params_save and (epoch % cfg.save_epoch == 0
or epoch == cfg.end_epoch - 1):
assert save_params is not None
params_path = get_epoch_params_filepath(cfg.model_name, epoch,
cfg.model_dir)
cfg.logger.info("save model params to %s, with select='%s'" %
(params_path, cfg.save_select))
save_params(params_path, net, select=cfg.save_select)
if epoch_lr_scheduler is not None:
epoch_lr_scheduler.step()
if final_reporthook is not None:
final_reporthook()
# coding: utf-8 # 2020/1/3 @ tongshiwei from longling import config_logging config_logging(logger="CangJie", console_log_level="info") # These constants are from gluonnlp UNK_TOKEN = '<unk>' BOS_TOKEN = '<bos>' EOS_TOKEN = '<eos>' PAD_TOKEN = '<pad>'
def numerical_check(_net, _cfg: Configuration, train_data, test_data, dump_result=False,
reporthook=None, final_reporthook=None): # pragma: no cover
ctx = _cfg.ctx
_net = set_device(_net, ctx)
bp_loss_f = get_bp_loss(ctx, **_cfg.loss_params)
loss_function = {}
loss_function.update(bp_loss_f)
from longling.ML.toolkit import EpochEvalFMT as Formatter
from longling.ML.toolkit import MovingLoss
from tqdm import tqdm
loss_monitor = MovingLoss(loss_function)
progress_monitor = tqdm
if dump_result:
from longling import config_logging
validation_logger = config_logging(
filename=path_append(_cfg.model_dir, "result.log"),
logger="%s-validation" % _cfg.model_name,
mode="w",
log_format="%(message)s",
)
evaluation_formatter = Formatter(
logger=validation_logger,
dump_file=_cfg.validation_result_file,
)
else:
evaluation_formatter = Formatter()
# train check
from longling.ML.PytorchHelper.toolkit.optimizer import get_trainer
trainer = get_trainer(
_net, optimizer=_cfg.optimizer,
optimizer_params=_cfg.optimizer_params,
select=_cfg.train_select
)
for epoch in range(_cfg.begin_epoch, _cfg.end_epoch):
for batch_data in progress_monitor(train_data, "Epoch: %s" % epoch):
fit_f(
net=_net, batch_data=batch_data,
trainer=trainer, bp_loss_f=bp_loss_f,
loss_function=loss_function,
loss_monitor=loss_monitor,
)
if epoch % 1 == 0:
msg, data = evaluation_formatter(
epoch=epoch,
loss_name_value=dict(loss_monitor.items()),
eval_name_value=eval_f(_net, test_data, ctx=ctx),
extra_info=None,
dump=dump_result,
)
print(msg)
if reporthook is not None:
reporthook(data)
# optional, whether reset the loss at the end of each epoch
loss_monitor.reset()
if final_reporthook is not None:
final_reporthook()
def __init__(self, logger=None):
self._initial_score = None
self._final_score = None
self._learner = None
self._logger = logger if logger else config_logging(logger="Tester")
# coding: utf-8
# create by tongshiwei on 2019/7/2
import codecs
import csv
import json
import networkx as nx
import pandas
from longling import wf_open, config_logging
from tqdm import tqdm
logger = config_logging(logger="junyi", console_log_level="info")
def build_ku_dict(source, target):
with codecs.open(source, encoding="utf-8") as f, wf_open(target) as wf:
f.readline()
idx = 0
vertex_dict = {}
for line in tqdm(csv.reader(f)):
if line[0] not in vertex_dict:
vertex_dict[line[0]] = idx
idx += 1
logger.info("vertex num: %s" % len(vertex_dict))
json.dump(vertex_dict, wf, indent=2)
def extract_prerequisite(source, target, ku_dict):
"""in target: (A, B) means predecessor --> successor"""
with codecs.open(
# coding: utf-8 from longling import config_logging logger = config_logging(logger="spider", console_log_level="info")
def logger():
return config_logging(logger="test")
# coding: utf-8
# 2020/4/16 @ tongshiwei
import functools
from pathlib import PurePath
import json
import math
import numpy as np
from longling import loading as _loading, as_out_io, PATH_TYPE, json_load, close_io
from longling import AddPrinter, type_from_name, config_logging, as_list
from sklearn.utils.validation import check_random_state
from sklearn.model_selection import KFold
from longling import concurrent_pool
_logger = config_logging(logger="dataset", console_log_level="info")
loading = functools.partial(_loading, src_type="text")
def _get_s_points(length: int, ratio: list) -> list:
"""
Returns
-------
s_points: list of int
Examples
--------
>>> _get_s_points(10, [0.1, 0.9])
[0, 1, 10]
"""
cum_ratios = np.cumsum([0.0] + ratio)
s_points = [math.ceil(length * ratio) for ratio in cum_ratios]
# create by tongshiwei on 2019/7/2
__all__ = ["url_dict", "get_data"]
import os
from urllib.request import urlretrieve
import requests
from bs4 import BeautifulSoup
from longling import config_logging, LogLevel, path_append
# from longling.spider import download_data
from .utils import decompress
DEFAULT_DATADIR = path_append("../", "data/", to_str=True)
config_logging(logger="downloader", console_log_level=LogLevel.INFO)
prefix = 'http://base.ustc.edu.cn/data/'
url_dict = {
"assistment-2009-2010-skill":
"http://base.ustc.edu.cn/data/ASSISTment/2009_skill_builder_data_corrected.zip",
"assistment-2012-2013-non-skill":
"http://base.ustc.edu.cn/data/ASSISTment/2012-2013-data-with-predictions-4-final.zip",
"assistment-2015":
"http://base.ustc.edu.cn/data/ASSISTment/2015_100_skill_builders_main_problems.zip",
"junyi":
"http://base.ustc.edu.cn/data/JunyiAcademy_Math_Practicing_Log/junyi.rar",
"KDD-CUP-2010": "http://base.ustc.edu.cn/data/KDD_Cup_2010/",
"slepemapy.cz": "http://base.ustc.edu.cn/data/slepemapy.cz/"
}
def json_logger(json_log_path):
return config_logging(filename=json_log_path,
file_format="%(message)s",
logger="test.json")
def numerical_check(_net,
_cfg: Configuration,
train_data,
test_data,
dump_result=False,
reporthook=None,
final_reporthook=None): # pragma: no cover
ctx = _cfg.ctx
batch_size = _cfg.batch_size
_net.initialize(ctx=ctx)
bp_loss_f = get_bp_loss(**_cfg.loss_params)
loss_function = {}
loss_function.update(bp_loss_f)
from longling.ML.MxnetHelper.glue import module
from longling.ML.toolkit import EpochEvalFMT as Formatter
from longling.ML.toolkit import MovingLoss
from tqdm import tqdm
loss_monitor = MovingLoss(loss_function)
progress_monitor = tqdm
if dump_result:
from longling import config_logging
validation_logger = config_logging(
filename=path_append(_cfg.model_dir, "result.log"),
logger="%s-validation" % _cfg.model_name,
mode="w",
log_format="%(message)s",
)
evaluation_formatter = Formatter(
logger=validation_logger,
dump_file=_cfg.validation_result_file,
)
else:
evaluation_formatter = Formatter()
# train check
trainer = module.Module.get_trainer(_net,
optimizer=_cfg.optimizer,
optimizer_params=_cfg.optimizer_params,
select=_cfg.train_select)
for epoch in range(_cfg.begin_epoch, _cfg.end_epoch):
for i, batch_data in enumerate(
progress_monitor(train_data, "Epoch: %s" % epoch)):
fit_f(
net=_net,
batch_size=batch_size,
batch_data=batch_data,
trainer=trainer,
bp_loss_f=bp_loss_f,
loss_function=loss_function,
loss_monitor=loss_monitor,
ctx=ctx,
)
if _cfg.lr_params and "update_params" in _cfg.lr_params:
_cfg.logger.info("reset trainer")
lr_params = _cfg.lr_params.pop("update_params")
lr_update_params = dict(batches_per_epoch=i + 1,
lr=_cfg.optimizer_params["learning_rate"],
update_epoch=lr_params.get(
"update_epoch",
_cfg.end_epoch - _cfg.begin_epoch - 1))
lr_update_params.update(lr_params)
trainer = module.Module.get_trainer(
_net,
optimizer=_cfg.optimizer,
optimizer_params=_cfg.optimizer_params,
lr_params=lr_update_params,
select=_cfg.train_select,
logger=_cfg.logger)
if epoch % 1 == 0:
msg, data = evaluation_formatter(iteration=epoch,
loss_name_value=dict(
loss_monitor.items()),
eval_name_value=eval_f(_net,
test_data,
ctx=ctx),
extra_info=None,
dump=dump_result,
keep={"msg", "data"})
print(msg)
if reporthook is not None:
reporthook(data)
if final_reporthook is not None:
final_reporthook()
# coding: utf-8 # Copyright @tongshiwei from longling import config_logging config_logging(logger="RFAG", console_log_level="info") from .RFAG import RFAG __all__ = ["RFAG"]
