def _update(self, **kwargs):
params = kwargs
params["logger"] = params.pop(
"logger",
config_logging(logger=params.get("model_name", self.model_name),
console_log_level="info"))
for key in params:
if key.endswith("_params") and key + "_update" in params:
params[key].update(params[key + "_update"])
self.deep_update(**params)
_vars = ["ctx"]
for _var in _vars:
if _var in kwargs:
try:
setattr(self, _var, eval_var(kwargs[_var]))
except TypeError:
pass
self.validation_result_file = path_append(self.model_dir,
RESULT_JSON,
to_str=True)
self.cfg_path = path_append(self.model_dir, CFG_JSON, to_str=True)
def toolbox_init(
self,
evaluation_formatter_parameters=None,
validation_logger_mode="w",
informer_silent=False,
):
from longling.lib.clock import Clock
from longling.lib.utilog import config_logging
from longling.ML.toolkit.formatter import MultiClassEvalFormatter \
as Formatter
from longling.ML.toolkit.monitor import MovingLoss, \
ConsoleProgressMonitor as ProgressMonitor
self.toolbox = {
"monitor": dict(),
"timer": None,
"formatter": dict(),
}
mod = self.mod
cfg = self.mod.cfg
# 4.1 todo 定义损失函数
# bp_loss_f 定义了用来进行 back propagation 的损失函数，
# 有且只能有一个，命名中不能为 *_\d+ 型
assert self.loss_function is not None
loss_monitor = MovingLoss(self.loss_function)
# 4.1 todo 初始化一些训练过程中的交互信息
timer = Clock()
progress_monitor = ProgressMonitor(
loss_index=[name for name in self.loss_function],
end_epoch=cfg.end_epoch - 1,
silent=informer_silent)
validation_logger = config_logging(
filename=os.path.join(cfg.model_dir, "result.log"),
logger="%s-validation" % cfg.model_name,
mode=validation_logger_mode,
log_format="%(message)s",
)
# set evaluation formatter
evaluation_formatter_parameters = {} \
if evaluation_formatter_parameters is None \
else evaluation_formatter_parameters
evaluation_formatter = Formatter(
logger=validation_logger,
dump_file=mod.cfg.validation_result_file,
**evaluation_formatter_parameters)
self.toolbox["monitor"]["loss"] = loss_monitor
self.toolbox["monitor"]["progress"] = progress_monitor
self.toolbox["timer"] = timer
self.toolbox["formatter"]["evaluation"] = evaluation_formatter
def toolbox_init(
self,
evaluation_formatter_parameters=None,
validation_logger_mode="w",
silent=False,
):
from longling import path_append
from longling.lib.clock import Clock
from longling.lib.utilog import config_logging
from longling.ML.toolkit import EpochEvalFMT as Formatter
from longling.ML.toolkit import MovingLoss, ConsoleProgressMonitor as ProgressMonitor
self.toolbox = {
"monitor": dict(),
"timer": None,
"formatter": dict(),
}
mod = self.mod
cfg = self.mod.cfg
assert self.loss_function is not None
loss_monitor = MovingLoss(self.loss_function)
# 4 todo 初始化一些训练过程中的交互信息
timer = Clock()
progress_monitor = ProgressMonitor(
indexes={"Loss": [name for name in self.loss_function]},
values={"Loss": loss_monitor.losses},
silent=silent,
player_type="epoch",
total_epoch=cfg.end_epoch - 1)
validation_logger = config_logging(
filename=path_append(cfg.model_dir, "result.log"),
logger="%s-validation" % cfg.model_name,
mode=validation_logger_mode,
log_format="%(message)s",
)
# set evaluation formatter
evaluation_formatter_parameters = {} \
if evaluation_formatter_parameters is None \
else evaluation_formatter_parameters
evaluation_formatter = Formatter(
logger=validation_logger,
dump_file=mod.cfg.validation_result_file,
**evaluation_formatter_parameters)
self.toolbox["monitor"]["loss"] = loss_monitor
self.toolbox["monitor"]["progress"] = progress_monitor
self.toolbox["timer"] = timer
self.toolbox["formatter"]["evaluation"] = evaluation_formatter
def _update(self, **kwargs):
params = kwargs
params["logger"] = params.pop(
"logger",
config_logging(logger=params.get("model_name", self.model_name),
console_log_level="info"))
for key in params:
if key.endswith("_params") and key + "_update" in params:
params[key].update(params[key + "_update"])
# path_override_check
path_check_list = [
"dataset", "root_data_dir", "workspace", "root_model_dir",
"model_dir"
]
_overridden = {}
for path_check in path_check_list:
if kwargs.get(path_check) is None or kwargs[path_check] == getattr(
self, "%s" % path_check):
_overridden[path_check] = False
else:
_overridden[path_check] = True
for param, value in params.items():
setattr(self, "%s" % param, value)
def is_overridden(varname):
return _overridden["%s" % varname]
# set dataset
if is_overridden("dataset") and not is_overridden("root_data_dir"):
kwargs["root_data_dir"] = path_append("$root", "data", "$dataset")
# set workspace
if (is_overridden("workspace") or is_overridden("root_model_dir")
) and not is_overridden("model_dir"):
kwargs["model_dir"] = path_append("$root_model_dir", "$workspace")
# rebuild relevant directory or file path according to the kwargs
_dirs = [
"workspace", "root_data_dir", "data_dir", "root_model_dir",
"model_dir"
]
for _dir in _dirs:
exp = var2exp(kwargs.get(_dir, getattr(self, _dir)),
env_wrap=lambda x: "self.%s" % x)
setattr(self, _dir, eval(exp))
self.validation_result_file = path_append(self.model_dir,
RESULT_JSON,
to_str=True)
self.cfg_path = path_append(self.model_dir, CFG_JSON, to_str=True)
def _update(self, **kwargs):
params = kwargs
params["logger"] = params.pop(
"logger",
config_logging(logger=params.get("model_name", self.model_name),
console_log_level="info"))
for key in params:
if key.endswith("_params") and key + "_update" in params:
params[key].update(params[key + "_update"])
self.deep_update(**params)
self.validation_result_file = path_append(self.model_dir,
RESULT_JSON,
to_str=True)
self.cfg_path = path_append(self.model_dir, CFG_JSON, to_str=True)
# coding: utf-8
# create by tongshiwei on 2019-9-4
from __future__ import absolute_import
from __future__ import print_function
import argparse
import os
from longling.lib.parser import path_append
from longling.lib.stream import wf_open
from longling.lib.utilog import config_logging, LogLevel
logger = config_logging(logger="glue",
level=LogLevel.DEBUG,
console_log_level=LogLevel.DEBUG)
def new_model(model_name,
source_dir,
directory=None,
level="project",
skip_existing=False): # pragma: no cover
target_dir = os.path.join(
directory, model_name) if directory is not None else model_name
target_dir = os.path.abspath(target_dir)
if level == "project":
pass
elif level == "model":
source_dir = path_append(
def __init__(self, params_path=None, **kwargs):
"""
Configuration File, including categories:
* directory setting
* optimizer setting
* training parameters
* equipment
* parameters saving setting
* user parameters
Parameters
----------
params_path: str
The path to configuration file which is in json format
kwargs:
Parameters to be reset.
"""
super(Configuration, self).__init__(
logger=config_logging(
logger=self.model_name,
console_log_level=LogLevel.INFO
)
)
params = self.class_var
if params_path:
params.update(self.load_cfg(cfg_path=params_path))
params.update(**kwargs)
for key in params:
if key.endswith("_params") and key + "_update" in params:
params[key].update(params[key + "_update"])
# path_override_check
path_check_list = ["dataset", "root_data_dir", "workspace", "root_model_dir", "model_dir"]
_overridden = {}
for path_check in path_check_list:
if kwargs.get(path_check) is None or kwargs[path_check] == getattr(self, "%s" % path_check):
_overridden[path_check] = False
else:
_overridden[path_check] = True
for param, value in params.items():
setattr(self, "%s" % param, value)
def is_overridden(varname):
return _overridden["%s" % varname]
# set dataset
if is_overridden("dataset") and not is_overridden("root_data_dir"):
kwargs["root_data_dir"] = path_append("$root", "data", "$dataset")
# set workspace
if (is_overridden("workspace") or is_overridden("root_model_dir")) and not is_overridden("model_dir"):
kwargs["model_dir"] = path_append("$root_model_dir", "$workspace")
# rebuild relevant directory or file path according to the kwargs
_dirs = [
"workspace", "root_data_dir", "data_dir", "root_model_dir",
"model_dir"
]
for _dir in _dirs:
exp = var2exp(
kwargs.get(_dir, getattr(self, _dir)),
env_wrap=lambda x: "self.%s" % x
)
setattr(self, _dir, eval(exp))
_vars = [
"ctx"
]
for _var in _vars:
if _var in kwargs:
try:
setattr(self, _var, eval_var(kwargs[_var]))
except TypeError:
pass
self.validation_result_file = path_append(
self.model_dir, "result.json", to_str=True
)
self.cfg_path = path_append(
self.model_dir, "configuration.json", to_str=True
)
def __init__(self, params_json=None, **kwargs):
"""
Configuration File, including categories:
* directory setting
* optimizer setting
* training parameters
* equipment
* parameters saving setting
* user parameters
Parameters
----------
params_json: str
The path to configuration file which is in json format
kwargs:
Parameters to be reset.
"""
super(Configuration, self).__init__(logger=config_logging(
logger=self.model_name, console_log_level=LogLevel.INFO))
params = self.class_var
if params_json:
params.update(self.load_cfg(params_json=params_json))
params.update(**kwargs)
for param, value in params.items():
setattr(self, "%s" % param, value)
# set dataset
if kwargs.get("dataset") and not kwargs.get("root_data_dir"):
kwargs["root_data_dir"] = "$root/data/$dataset"
# set workspace
if (kwargs.get("workspace") or
kwargs.get("root_model_dir")) and not kwargs.get("model_dir"):
kwargs["model_dir"] = "$root_model_dir/$workspace"
# rebuild relevant directory or file path according to the kwargs
_dirs = [
"workspace", "root_data_dir", "data_dir", "root_model_dir",
"model_dir"
]
for _dir in _dirs:
exp = var2exp(kwargs.get(_dir, getattr(self, _dir)),
env_wrap=lambda x: "self.%s" % x)
setattr(self, _dir, eval(exp))
_vars = ["ctx"]
for _var in _vars:
if _var in kwargs:
try:
setattr(self, _var, eval_var(kwargs[_var]))
except TypeError:
pass
self.validation_result_file = path_append(self.model_dir,
"result.json",
to_str=True)
self.cfg_path = path_append(self.model_dir,
"configuration.json",
to_str=True)
# coding: utf-8
from __future__ import absolute_import
import logging
import time
from contextlib import contextmanager
from longling.lib.utilog import config_logging
_logger = config_logging(logger="clock", console_log_level=logging.INFO)
__all__ = ["Clock", "print_time", "Timer"]
@contextmanager
def print_time(tips: str = "", logger=_logger):
"""
统计并打印脚本运行时间， 秒为单位
Parameters
----------
tips: str
logger: logging.Logger or logging
Examples
--------
>>> with print_time("tips"):
... a = 1 + 1 # The code you want to test
"""
start_time = time.time()
import functools
import os
from collections import OrderedDict
from shutil import copyfile as _copyfile, rmtree, copytree as _copytree
from longling import wf_open, PATH_TYPE
from longling.Architecture.utils import binary_legal_input
from longling.lib.path import abs_current_dir, path_append
from longling.lib.regex import default_variable_replace as dvr
from longling.lib.utilog import config_logging, LogLevel
from longling.lib.yaml_helper import FoldedString, ordered_yaml_load, dump_folded_yaml
from . import config
logger = config_logging(logger="arch", console_log_level=LogLevel.INFO)
META = path_append(abs_current_dir(__file__), "meta_docs")
default_variable_replace = functools.partial(dvr, quotation="\'")
def copytree(src, dst, **kwargs):
"""
Recursively copy a directory tree.
Change OVERRIDE mode to specify the operation when dst existed.
Examples
--------
.. code-block :: python
from __future__ import absolute_import
from __future__ import division
import logging
import os
from collections import OrderedDict
from tqdm import tqdm
import requests
from longling.lib.utilog import config_logging
from longling.lib.stream import check_file, wf_open, wf_close
from dev.spider import conf
logger = config_logging(logger="spider", console_log_level=logging.INFO, propagate=False)
COOKIES = conf.cookies
HEADERS = conf.headers
def conf_request(url, cookies=conf.cookies, headers=conf.headers):
return requests.get(url, cookies=cookies, headers=headers)
def dyn_ua_requests(urls, cookies=conf.cookies, ua_list="doc/agent_list"):
import random
with open(ua_list) as f:
uas = [line.strip() for line in f if '#' not in line]
for url in urls:
def get_default_toolbox(loss_function=None,
evaluation_formatter_parameters=None,
progress_monitor_parameters=None,
validation_logger_mode="w",
silent=False,
configuration=None): # pragma: no cover
"""
New in version 1.3.16
todo: consider whether to keep it
Notice
------
The developer who modify this document should simultaneously modify the related function in glue
"""
from longling import path_append
from longling.lib.clock import Clock
from longling.lib.utilog import config_logging
from longling.ML.toolkit import EpochEvalFMT as Formatter
from longling.ML.toolkit import MovingLoss, ConsoleProgressMonitor as ProgressMonitor
cfg = configuration
toolbox = {
"monitor": dict(),
"timer": None,
"formatter": dict(),
}
loss_monitor = MovingLoss(loss_function) if loss_function else None
timer = Clock()
progress_monitor = ProgressMonitor(
indexes={"Loss": [name
for name in loss_function]} if loss_function else {},
values={"Loss": loss_monitor.losses} if loss_monitor else {},
silent=silent,
**progress_monitor_parameters
if progress_monitor_parameters is not None else {})
validation_logger = config_logging(
filename=path_append(cfg.model_dir, "result.log") if hasattr(
cfg, "model_dir") else None,
logger="%s-validation" %
cfg.model_name if hasattr(cfg, "model_name") else "model",
mode=validation_logger_mode,
log_format="%(message)s",
)
# set evaluation formatter
evaluation_formatter_parameters = {} \
if evaluation_formatter_parameters is None \
else evaluation_formatter_parameters
evaluation_formatter = Formatter(logger=validation_logger,
dump_file=getattr(
cfg, "validation_result_file", False),
**evaluation_formatter_parameters)
toolbox["monitor"]["loss"] = loss_monitor
toolbox["monitor"]["progress"] = progress_monitor
toolbox["timer"] = timer
toolbox["formatter"]["evaluation"] = evaluation_formatter
return toolbox
