def __init__(self, conn, table = None):
self.conn = conn
self.cursor = conn.cursor()
self.table = table if table.startswith(config.MYSQL_PRE) else \
config.MYSQL_PRE + table
self.commit = False
self.fields = self.get_table_fields()
self.logger = get_logger()
def __init__(self):
self.logger = get_logger()
try:
version = self.option.version
except:
version = self.__version__
version = float(version) if version else 0
if version < self.__version__:
self.update(version)
self.option.version = self.__version__
def __init__(self):
self.logger = get_logger()
try:
version = self.option.version
except:
version = self.__version__
version = float(version) if version else 0
if version < self.__version__:
self.update(version)
self.option.version = self.__version__
def main(port=None):
port = port if port else PORT
logger = get_logger()
info = ""
info += "Running On Port: %s\n" % port
apps = load_apps()
app_list = list(apps.items())
app_list.append((r"(/.*)", ErrorHandler))
for a in app_list:
info += "\t%-20s %s\n" % (a[0], a[1])
logger.info(info)
if settings.get("debug", False):
logger.warn("Debug Mode is On")
app = Application(app_list, **settings)
app.listen(port)
IOLoop.instance().start()
def main(port=None):
port = port if port else PORT
logger = get_logger()
info = ''
info += 'Running On Port: %s\n' % port
apps = load_apps()
app_list = list(apps.items())
app_list.append((r"(/.*)", ErrorHandler))
for a in app_list:
info += '\t%-20s %s\n' % (a[0], a[1])
logger.info(info)
if settings.get('debug', False):
logger.warn('Debug Mode is On')
app = Application(app_list, **settings)
app.listen(port)
IOLoop.instance().start()
def __init__(
self,
log_path="log", #
console_log=True, # 屏幕打印日志开关，默认True
console_log_level=logging.INFO, # 屏幕打印日志的级别，默认为INFO
critical_log=False, # critica单独l写文件日志，默认关闭
error_log=True, # error级别单独写文件日志，默认开启
warning_log=False, # warning级别单独写日志，默认关闭
info_log=True, # info级别单独写日志，默认开启
debug_log=False, # debug级别日志，默认关闭
**kwargs):
self.logger = util.get_logger(
log_path=log_path, #
console_log=console_log, # 屏幕打印日志开关，默认True
console_log_level=console_log_level, # 屏幕打印日志的级别，默认为INFO
critical_log=critical_log, # critica单独l写文件日志，默认关闭
error_log=error_log, # error级别单独写文件日志，默认开启
warning_log=warning_log, # warning级别单独写日志，默认关闭
info_log=info_log, # info级别单独写日志，默认开启
debug_log=debug_log, # debug级别日志，默认关闭
logger_name=self.__class__.__name__,
)
def __on_start__(self):
"""
进程开始运行时调用
:return:
"""
if self.check_prerequisites() is not True:
sys.exit(0)
self.init_redis()
# 实例化self.logger
self.logger = util.get_logger(
logger_name=self.__class__.__name__,
log_path=self.__log_path__, #
console_log=self.__console_log__, # 屏幕打印日志开关，默认True
console_log_level=self.__console_log_level__, # 屏幕打印日志的级别，默认为INFO
critical_log=self.__critical_log__, # critical写文件日志，默认关闭
error_log=self.__error_log__, # error级别单独写文件日志，默认开启
warning_log=self.__warning_log__, # warning级别单独写日志，默认关闭
info_log=self.__info_log__, # info级别单独写日志，默认开启
debug_log=self.__debug_log__, # debug级别日志，默认关闭
)
import argparse
from sklearn.metrics import f1_score
from core.loader import DataLoader
from core.util import timer, get_logger
from core.model import lr_model
from core.preprocessing import get_stopwords, tokenizer
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--exp", default="exp")
args = parser.parse_args()
logger = get_logger(exp=args.exp)
with timer("Load Data", logger):
loader = DataLoader()
with timer("tokenize", logger):
loader.tokenize(tokenizer, {
"stopwords": get_stopwords(),
"include_verb": True
})
train, test = loader.load()
X = train["tokenized"]
X_test = test["tokenized"]
y = train["label"]
y_test = test["label"]
model = lr_model()
def __init__(self):
self.logger = get_logger()
self.logger.info("UPLOAD_PATH %s", UPLOAD_PATH)
if not os.path.exists(self.root):
self.logger.debug("create upload_dir %s", self.root)
os.makedirs(self.root)
def init_logger():
return util.get_logger(logger_name="console")
def __init__(
self,
singleton=True, # 单例模式
nickname=None, # Worker的自定义名字
description="No Description", # 备注说明
heart_beat_interval=1, # 默认1秒心跳
log_path="log", #
console_log=True, # 屏幕打印日志开关，默认True
console_log_level=logging.INFO, # 屏幕打印日志的级别，默认为INFO
critical_log=False, # critical单独写文件日志，默认关闭
error_log=True, # error级别单独写文件日志，默认开启
warning_log=False, # warning级别单独写日志，默认关闭
info_log=True, # info级别单独写日志，默认开启
debug_log=False, # debug级别日志，默认关闭
**kwargs):
super().__init__()
# 记录日志配置
self.__log_path__ = log_path
self.__console_log__ = console_log
self.__console_log_level__ = console_log_level
self.__critical_log__ = critical_log
self.__error_log__ = error_log
self.__warning_log__ = warning_log
self.__info_log__ = info_log
self.__debug_log__ = debug_log
self.logger = util.get_logger(
logger_name=self.__class__.__name__,
log_path=self.__log_path__,
console_log=self.__console_log__, # 屏幕打印日志开关，默认True
console_log_level=self.__console_log_level__, # 屏幕打印日志的级别，默认为INFO
critical_log=self.__critical_log__, # critica单独l写文件日志，默认关闭
error_log=self.__error_log__, # error级别单独写文件日志，默认开启
warning_log=self.__warning_log__, # warning级别单独写日志，默认关闭
info_log=self.__info_log__, # info级别单独写日志，默认开启
debug_log=self.__debug_log__, # debug级别日志，默认关闭
)
self.__token__ = util.generate_token()
if nickname is None:
self.__nickname__ = self.__class__.__name__
else:
self.__nickname__ = nickname
self.nickname = self.__nickname__
self.name = self.__nickname__
self.__singleton__ = singleton
self.__description__ = description
self.__heart_beat_interval__ = heart_beat_interval
self.__threads__ = dict() # 被监控的线程
self.__data_feeder__ = set() # 本Worker订阅的内容
self.__follower__ = set() # Follower
self.__error_msg__ = None #
self.__stop_info__ = None #
self.__stop_time__ = None #
self.__status__ = "init"
self.redis_key = "HySAS.Worker." + \
self.__class__.__name__ + "." + self.__nickname__ + "."
self.channel_pub = self.redis_key + "Pub"
"""
self.__threads__ = {
"nickname": {
"description" : "该线程功能备注说明",
"name" : "该线程的名字",
"target" : "该线程的target"
"restart_mode" : "重启模式，可以为 manual/auto/remove;
manual则代表允许管理员发送命令手工重启线程,
auto则一旦线程关闭立即自动开启，
remove则代表一旦线程结束就从监控列表移除",
"restart_func" : "自动/手动重启时调用的方法",
},
}
"""
self.shutdown_signals = [
"SIGQUIT", # quit 信号
"SIGINT", # 键盘信号
"SIGHUP", # nohup 命令
"SIGTERM", # kill 命令
]
for s in self.shutdown_signals:
# 捕获退出信号后的要调用的,唯一的 shutdown 接口
try:
if hasattr(signal, s):
signal.signal(getattr(signal, s, None),
self.__on_termination__)
except Exception as e:
self.logger.info("绑定退出信号：{}失败，可能与windows系统有关。".format(s))
# 清空它，在run以后重新实例化
# 否则windows下会无法pickle
del (self.logger)
def __init__(self):
self.logger = get_logger()
self.logger.info("UPLOAD_PATH %s", UPLOAD_PATH)
if not os.path.exists(self.root):
self.logger.debug("create upload_dir %s", self.root)
os.makedirs(self.root)
