def remove_config(cls, qa_manager: QAManager, logger: Logger):
"""
清空所有配置
@param {QAManager} qa_manager - 数据管理对象
@param {Logger} logger - 日志对象
"""
# 标准问题和答案
_std_q = StdQuestion.get_or_none(
StdQuestion.tag == 'form_direct_action')
if _std_q is not None:
# 删除对应问题的意图
_ret = NlpPurposConfigDict.delete().where(
NlpPurposConfigDict.std_question_id == _std_q.id).execute()
if logger is not None:
logger.debug('remove form plugin nlp config success: %s !' %
str(_ret))
_ret = Answer.delete().where(
Answer.std_question_id == _std_q.id).execute()
_ret = StdQuestion.delete().where(
StdQuestion.id == _std_q.id).execute()
if logger is not None:
logger.debug(
'remove form plugin std question config success: %s !' %
str(_ret))
def use_simple_queue(logger: Logger):
"""
使用simple_log输出内存队列
@param {HiveNetLib.simple_log.Logger} logger - 日志对象
"""
_index = 0
while _index < RANGE_NUM:
_index += 1
logger.info('use simple queue index %d !' % _index)
def use_simple_console(logger: Logger):
"""
使用simple_log输出屏幕
@param {HiveNetLib.simple_log.Logger} logger - 日志对象
"""
_index = 0
while _index < RANGE_NUM:
_index += 1
logger.info('use simple console index %d !' % _index)
def remove_data(cls, qa_manager: QAManager, logger: Logger):
"""
清空所有数据
@param {QAManager} qa_manager - 数据管理对象
@param {Logger} logger - 日志对象
"""
AnswerDao.drop_tables(DATA_TABLES)
AnswerDao.create_tables(DATA_TABLES)
if logger is not None:
logger.debug('remove leave message plugin data success!')
def get_init_objs(cls) -> dict:
"""
获取工具初始化对象
@returns {dict} - 初始化对象
"""
# 获取配置文件信息
_execute_path = os.path.realpath(FileTool.get_file_path(__file__))
_config = os.path.join(_execute_path, '../conf/server.xml')
_config_xml = SimpleXml(_config, encoding='utf-8')
_server_config = _config_xml.to_dict()['server']
# 日志对象
_logger: Logger = None
if 'logger' in _server_config.keys():
_logger = Logger.create_logger_by_dict(_server_config['logger'])
# 连接数据库操作对象
_qa_manager = QAManager(
_server_config['answerdb'],
_server_config['milvus'],
_server_config['bert_client'],
logger=_logger,
excel_batch_num=_server_config['excel_batch_num'],
excel_engine=_server_config['excel_engine'],
load_para=False)
return {'logger': _logger, 'qa_manager': _qa_manager}
def remove_data(cls, qa_manager: QAManager, logger: Logger):
"""
清空所有数据
@param {QAManager} qa_manager - 数据管理对象
@param {Logger} logger - 日志对象
"""
AnswerDao.drop_tables(DATA_TABLES)
AnswerDao.create_tables(DATA_TABLES)
# 删除上传文件
_execute_path = os.path.realpath(FileTool.get_file_path(__file__))
FileTool.remove_all_with_path(
path=os.path.join(_execute_path, '../client/static/leave_message'))
if logger is not None:
logger.debug('remove leave message plugin data success!')
def init_pipeline_plugins():
"""
装载管道插件
"""
# 装载配置
_execute_path = os.path.realpath(os.path.join(
os.path.dirname(__file__), os.path.pardir, 'search_by_image'
))
RunTool.set_global_var('EXECUTE_PATH', _execute_path)
_config = os.path.join(_execute_path, 'conf/server_jade.xml')
_config_xml = SimpleXml(_config, encoding='utf-8')
_server_config = _config_xml.to_dict()['server']
RunTool.set_global_var(
'PIPELINE_PROCESSER_PARA', _server_config['pipeline']['processer_para']
)
RunTool.set_global_var('PIPELINE_ROUTER_PARA', _server_config['pipeline']['router_para'])
_plugins_path_list = _server_config['pipeline']['plugins_path'].split(',')
for _plugins_path in _plugins_path_list:
Pipeline.load_plugins_by_path(
os.path.join(_execute_path, _plugins_path.strip())
)
_logger: Logger = None
if 'logger' in _server_config.keys():
_logger_config = _server_config['logger']
if len(_logger_config['conf_file_name']) > 0 and _logger_config['conf_file_name'][0] == '.':
# 相对路径
_logger_config['conf_file_name'] = os.path.join(
_execute_path, _logger_config['conf_file_name']
)
if len(_logger_config['logfile_path']) > 0 and _logger_config['logfile_path'][0] == '.':
# 相对路径
_logger_config['logfile_path'] = os.path.join(
_execute_path, _logger_config['logfile_path']
)
_logger = Logger.create_logger_by_dict(_logger_config)
# 创建空管道用于测试
_empty_pipeline = Pipeline('empty', '{}', logger=_logger)
RunTool.set_global_var('EMPTY_PIPELINE', _empty_pipeline)
# 创建测试管道
_jade_pipeline = Pipeline(
'jade_search',
_server_config['pipeline']['pipeline_config']['JadeSearch'],
logger=_logger
)
RunTool.set_global_var('JADE_PIPELINE', _jade_pipeline)
def test_methon(self):
"""
测试静态方法
"""
_logger = Logger()
_result = CResult('00000')
_result.net_info = None
with ExceptionTool.ignored_cresult(_result,
logger=_logger,
expect=(),
error_map={
ImportError: ('20401', None),
BlockingIOError: ('20407', None)
},
self_log_msg='test:',
force_log_level=None):
_result.test = 'test'
self.assertTrue(_result.code == '00000' and _result.test == 'test',
'ok result error')
_result = CResult('00000')
with ExceptionTool.ignored_cresult(_result,
logger=_logger,
expect=(),
error_map={
ImportError: ('20401', None),
BlockingIOError: ('20407', None)
},
self_log_msg='test:',
force_log_level=None):
raise ImportError
self.assertTrue(_result.code == '20401',
'20401 result error, code:' + _result.code)
_result = CResult('00000')
with ExceptionTool.ignored_cresult(_result,
logger=_logger,
expect=(),
error_map={
ImportError: ('20401', None),
BlockingIOError: ('20407', None)
},
self_log_msg='test:',
force_log_level=None):
raise BlockingIOError
self.assertTrue(_result.code == '20407',
'20407 result error, code:' + _result.code)
def __init__(self):
"""
初始化测试参数
"""
# 获取配置
self.execute_path = os.path.realpath(
os.path.join(os.path.dirname(__file__), os.path.pardir,
'search_by_image'))
RunTool.set_global_var('EXECUTE_PATH', self.execute_path)
_config = os.path.join(self.execute_path, 'conf/server.xml')
_config_xml = SimpleXml(_config, encoding='utf-8')
self.server_config = _config_xml.to_dict()['server']
# 装载管道插件
RunTool.set_global_var(
'PIPELINE_PROCESSER_PARA',
self.server_config['pipeline']['processer_para'])
RunTool.set_global_var('PIPELINE_ROUTER_PARA',
self.server_config['pipeline']['router_para'])
_plugins_path_list = self.server_config['pipeline'][
'plugins_path'].split(',')
for _plugins_path in _plugins_path_list:
Pipeline.load_plugins_by_path(
os.path.join(self.execute_path, _plugins_path.strip()))
# 日志对象
self.logger: Logger = None
if 'logger' in self.server_config.keys():
_logger_config = self.server_config['logger']
if len(_logger_config['conf_file_name']
) > 0 and _logger_config['conf_file_name'][0] == '.':
# 相对路径
_logger_config['conf_file_name'] = os.path.join(
self.execute_path, _logger_config['conf_file_name'])
if len(_logger_config['logfile_path']
) > 0 and _logger_config['logfile_path'][0] == '.':
# 相对路径
_logger_config['logfile_path'] = os.path.join(
self.execute_path, _logger_config['logfile_path'])
self.logger = Logger.create_logger_by_dict(_logger_config)
# 创建搜索引擎
self.search_engine = SearchEngine(self.server_config,
logger=self.logger)
def remove_config(cls, qa_manager: QAManager, logger: Logger):
"""
清空所有配置
@param {QAManager} qa_manager - 数据管理对象
@param {Logger} logger - 日志对象
"""
# 意图配置
_ret = NlpPurposConfigDict.delete().where(
NlpPurposConfigDict.action == 'leave_message').execute()
if logger is not None:
logger.debug(
'remove leave message plugin nlp config success: %s !' %
str(_ret))
# 文件上传参数
_ret = UploadFileConfig.delete().where(
UploadFileConfig.upload_type ==
LEAVE_MESSAGE_PLUGIN_UPLOAD_FILE_CONFIG['upload_type']).execute()
if logger is not None:
logger.debug(
'remove leave message plugin file upload config success: %s !'
% str(_ret))
# 标准问题和答案
_std_q = StdQuestion.get_or_none(
StdQuestion.tag == 'leave_message_direct_action')
if _std_q is not None:
_ret = Answer.delete().where(
Answer.std_question_id == _std_q.id).execute()
_ret = StdQuestion.delete().where(
StdQuestion.id == _std_q.id).execute()
if logger is not None:
logger.debug(
'remove leave message plugin std question config success: %s !'
% str(_ret))
def import_config(cls, qa_manager: QAManager, logger: Logger):
"""
添加标准配置(不考虑删除问题)
@param {QAManager} qa_manager - 数据管理对象
@param {Logger} logger - 日志对象
"""
FORM_PLUGIN_CONFIG = RunTool.get_global_var('FORM_PLUGIN_CONFIG')
if FORM_PLUGIN_CONFIG is None:
FORM_PLUGIN_CONFIG = dict()
RunTool.set_global_var('FORM_PLUGIN_CONFIG', FORM_PLUGIN_CONFIG)
FORM_PLUGIN_SELF_TABLE = RunTool.get_global_var(
'FORM_PLUGIN_SELF_TABLE')
if FORM_PLUGIN_SELF_TABLE is None:
FORM_PLUGIN_SELF_TABLE = dict()
RunTool.set_global_var('FORM_PLUGIN_SELF_TABLE',
FORM_PLUGIN_SELF_TABLE)
# 插入标准问题
_std_q = StdQuestion.create(tag='form_direct_action',
q_type='context',
milvus_id=-1,
collection=FORM_PLUGIN_COLLECTION,
partition=FORM_PLUGIN_PARTITION,
question='表单插件通用处理')
# 插入问题答案
Answer.create(std_question_id=_std_q.id,
a_type='job',
type_param="['FormPlugin', 'operate', {}]",
replace_pre_def='N',
answer='表单插件通用处理')
if logger is not None:
logger.info('create form plugin std question config success!')
# 处理扩展插件
if FORM_PLUGIN_SEARCH_PATH is not None:
_path = os.path.join(os.path.dirname(__file__),
FORM_PLUGIN_SEARCH_PATH)
_file_list = FileTool.get_filelist(path=_path,
regex_str=r'.*\.py$',
is_fullname=False)
for _file in _file_list:
if _file == '__init__.py':
continue
# 执行加载
_module = ImportTool.import_module(_file[0:-3],
extend_path=_path,
is_force=True)
_clsmembers = inspect.getmembers(_module, inspect.isclass)
for (_class_name, _class) in _clsmembers:
if _module.__name__ != _class.__module__:
# 不是当前模块定义的函数
continue
# 判断类型
_get_form_type = getattr(_class, 'get_form_type', None)
if _get_form_type is None or not callable(_get_form_type):
# 不是标准的插件类
continue
_form_type = _get_form_type()
_get_form_config = getattr(_class, 'get_form_config', None)
# 加入配置
FORM_PLUGIN_CONFIG[_form_type] = _get_form_config()
# 循环插件实例进行处理
for _form_type in FORM_PLUGIN_CONFIG.keys():
_config = FORM_PLUGIN_CONFIG[_form_type]
# 创建表单类型意图参数
NlpPurposConfigDict.create(
action=_form_type,
match_collection='',
match_partition='',
collection=FORM_PLUGIN_COLLECTION,
partition=FORM_PLUGIN_PARTITION,
std_question_id=_std_q.id,
order_num=_config['order_num'],
exact_match_words=str(_config['exact_match_words']),
exact_ignorecase=_config['exact_ignorecase'],
match_words=str(_config['match_words']),
ignorecase=_config['ignorecase'],
word_scale=_config['word_scale'],
info=str(_config['info']),
check=str(_config['check']))
if logger is not None:
logger.info('create form plugin [%s] success!' % _form_type)
@file simple_parallel_demo.py
"""
import sys
import os
import time
import multiprocessing
sys.path.append(os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
from HiveNetLib.simple_log import EnumLoggerName, Logger, EnumLoggerConfigType
from HiveNetLib.simple_parallel import ThreadParallel, ProcessParallel, ProcessParallelShareDict, ProcessParallelLock, ParallelPool
from HiveNetLib.base_tools.import_tool import ImportTool
# 通用的logger
_logger = Logger(conf_file_name=None,
logger_name=EnumLoggerName.Console,
config_type=EnumLoggerConfigType.JSON_STR)
# 通用的logger创建参数
_logger_kwargs = {
'conf_file_name': None,
'logger_name': EnumLoggerName.Console,
'config_type': EnumLoggerConfigType.JSON_STR
}
# 通用的并发任务处理函数
def demo_deal_fun(a, b, c, p1='', p2=''):
print('demo_deal_fun print: a=%s, b=%s, c=%s, p1=%s, p2=%s' %
(str(a), str(b), str(c), str(p1), str(p2)))
time.sleep(0.1)
def create_call_chain_logger(
logger=None, is_use_global_logger=True,
asyn_logging=False, asyn_log_config_level=logging.DEBUG,
topic_name='', asyn_logging_fun=None, asyn_deal_msg_fun=None, asyn_formater=None
):
"""
创建调用链日志对象
@param {simple_log.Logger} logger=None - 日志对象，如果为None代表不需要输出日志
@param {bool} is_use_global_logger=True - 当logger=None时，是否使用全局logger对象
注：通过RunTool.set_global_logger进行设置
@param {bool} asyn_logging=False - 是否异步写日志
@param {int} asyn_log_config_level=logging.DEBUG - 异步日志配置文件的级别
@param {string} topic_name='' - 日志主题
@param {function} asyn_logging_fun=None - 异步写日志时的日志处理函数，如果为None时将默认使用
logger传入的日志对象进行写处理（注意会修改对象的formater为'%(message)s'），
也可以自行传入一个处理函数实现远程日志的记录，
函数格式为funs(levelno, topic_name, msg){...}
@param {function} asyn_deal_msg_fun=None - 异步写日志时的日志msg生成函数（可以自定义函数处理
record.info_dict来生成日志msg），如果为None时默认直接当前的msg写日志
函数格式为funs(topic_name, record){return msg_string}，返回生成后的日志msg内容
@param {logging.Formatter} asyn_formater=None - 异步写日志的Formatter格式，如果传None，
代表使用logger的格式，logging.Formatter可传入日志的format格式字符串及datefmt日期时间格式
@return {simple_log.Logger} - 可进行调用链日志处理的日志对象
"""
_logger = None
if asyn_logging:
# 异步日志模式
if logger is not None:
# 有传入原始日志对象的情况下才能支持异步处理
_LOGGER_QUEUE_MSG_JSON_STR = u'''{
"version": 1,
"disable_existing_loggers": false,
"formatters": {
"simpleFormatter": {
"format": "[%(asctime)s.%(millisecond)s][%(levelname)s][PID:%(process)d][TID:%(thread)d][FILE:%(filename)s][FUN:%(funcName)s]%(message)s",
"datefmt": "%Y-%m-%d %H:%M:%S"
}
},
"handlers": {
"QueueRecordHandler": {
"class": "HiveNetLib.simple_log.QueueHandler",
"level": "DEBUG",
"formatter": "simpleFormatter",
"queue": "",
"topic_name": "{=topic_name=}",
"is_deal_msg": false,
"error_queue_size": 20
}
},
"loggers": {
"QueueRecord": {
"level": "DEBUG",
"handlers": ["QueueRecordHandler"],
"propagate": "no"
}
},
"root": {
"level": "DEBUG",
"handlers": []
}
}
'''.replace(
'DEBUG', logging.getLevelName(asyn_log_config_level)
).replace('{=topic_name=}', topic_name)
# 新增日志对象
_logger = Logger(
logger_name='QueueRecord',
json_str=_LOGGER_QUEUE_MSG_JSON_STR
)
# 部分信息基于对象本身属性处理
_logger._asyn_base_logger = logger
_logger._asyn_logging_fun = asyn_logging_fun
_logger._asyn_deal_msg_fun = asyn_deal_msg_fun
# 修改日志Formatter
_logger._formater = asyn_formater
if _logger._formater is None:
_logger._formater = copy.deepcopy(logger.get_logger_formater())
_logger.set_logger_formater(
_logger._formater
)
_logger._asyn_base_logger.set_logger_formater('%(message)s')
else:
_logger = logger
if logger is None and is_use_global_logger and not asyn_logging:
# 使用全局logger，注意异步模式将不使用全局日志对象
_logger = RunTool.get_global_logger()
if _logger is not None and asyn_deal_msg_fun is not None:
_logger._asyn_deal_msg_fun = asyn_deal_msg_fun
_logger._topic_name = topic_name
# 返回日志对象
return _logger
def setUpClass(cls):
print("test class start =======>")
cls._logger = Logger()
def import_config(cls, qa_manager: QAManager, logger: Logger):
"""
添加标准配置(不考虑删除问题)
@param {QAManager} qa_manager - 数据管理对象
@param {Logger} logger - 日志对象
"""
# 插入标准问题
_std_q = StdQuestion.create(tag='leave_message_direct_action',
q_type='context',
milvus_id=-1,
collection=LEAVE_MESSAGE_PLUGIN_COLLECTION,
partition=LEAVE_MESSAGE_PLUGIN_PARTITION,
question='留言插件通用处理')
# 插入问题答案
Answer.create(
std_question_id=_std_q.id,
a_type='ask',
type_param="['LeaveMessagePlugin', 'save_msg', '', '', {}, True]",
replace_pre_def='N',
answer='留言插件通用处理')
if logger is not None:
logger.info(
'create leave message plugin std question config success!')
# 创建留言意图参数
NlpPurposConfigDict.create(
action='leave_message',
match_collection='',
match_partition='',
collection=LEAVE_MESSAGE_PLUGIN_COLLECTION,
partition=LEAVE_MESSAGE_PLUGIN_PARTITION,
std_question_id=_std_q.id,
order_num=LEAVE_MESSAGE_PLUGIN_NLP_CONFIG['order_num'],
exact_match_words=str(
LEAVE_MESSAGE_PLUGIN_NLP_CONFIG['exact_match_words']),
exact_ignorecase=LEAVE_MESSAGE_PLUGIN_NLP_CONFIG[
'exact_ignorecase'],
match_words=str(LEAVE_MESSAGE_PLUGIN_NLP_CONFIG['match_words']),
ignorecase=LEAVE_MESSAGE_PLUGIN_NLP_CONFIG['ignorecase'],
word_scale=LEAVE_MESSAGE_PLUGIN_NLP_CONFIG['word_scale'],
info=str(LEAVE_MESSAGE_PLUGIN_NLP_CONFIG['info']),
check=str(LEAVE_MESSAGE_PLUGIN_NLP_CONFIG['check']))
if logger is not None:
logger.info('create leave message plugin nlp config success!')
# 创建文件上传参数
UploadFileConfig.create(
upload_type=LEAVE_MESSAGE_PLUGIN_UPLOAD_FILE_CONFIG['upload_type'],
exts=LEAVE_MESSAGE_PLUGIN_UPLOAD_FILE_CONFIG['exts'],
size=LEAVE_MESSAGE_PLUGIN_UPLOAD_FILE_CONFIG['size'],
save_path=LEAVE_MESSAGE_PLUGIN_UPLOAD_FILE_CONFIG['save_path'],
url=LEAVE_MESSAGE_PLUGIN_UPLOAD_FILE_CONFIG['url'],
rename=LEAVE_MESSAGE_PLUGIN_UPLOAD_FILE_CONFIG['rename'],
after=LEAVE_MESSAGE_PLUGIN_UPLOAD_FILE_CONFIG['after'],
remark=LEAVE_MESSAGE_PLUGIN_UPLOAD_FILE_CONFIG['remark'])
if logger is not None:
logger.info(
'create leave message plugin upload file config success!')
del _logger
# 使用日志对象输出到文件
_logger = logging.getLogger(name='File')
_start = datetime.datetime.now()
use_logging_file(_logger)
_end = datetime.datetime.now()
_logging_file_time = (_end - _start).total_seconds()
del _logger
# 使用simple_log日志对象输出屏幕
SIMPLE_LOGGER_CONFIG['handlers']['FileHandler'][
'filename'] = LOGGER_CONFIG['handlers']['FileHandler']['filename']
_simple_json_config = json.dumps(SIMPLE_LOGGER_CONFIG, ensure_ascii=False)
_logger = Logger(logger_name='Console',
json_str=_simple_json_config,
is_create_logfile_by_day=False)
_start = datetime.datetime.now()
use_simple_console(_logger)
_end = datetime.datetime.now()
_simple_console_time = (_end - _start).total_seconds()
del _logger
# 使用simple_log日志对象输出文件
_logger = Logger(logger_name='File',
json_str=_simple_json_config,
is_create_logfile_by_day=False)
_start = datetime.datetime.now()
use_simple_file(_logger)
_end = datetime.datetime.now()
_simple_file_time = (_end - _start).total_seconds()
def __init__(self, server_config: dict, app: Flask = None, **kwargs):
"""
初始化QA问答服务
@param {dict} server_config - 服务配置字典
@param {Flask} app=None - 服务
"""
self.debug = server_config.get('debug', True)
self.execute_path = server_config['execute_path']
# 日志处理
self.logger: Logger = None
if 'logger' in server_config.keys():
_logger_config = server_config['logger']
if len(_logger_config['conf_file_name']
) > 0 and _logger_config['conf_file_name'][0] == '.':
# 相对路径
_logger_config['conf_file_name'] = os.path.join(
self.execute_path, _logger_config['conf_file_name'])
if len(_logger_config['logfile_path']
) > 0 and _logger_config['logfile_path'][0] == '.':
# 相对路径
_logger_config['logfile_path'] = os.path.join(
self.execute_path, _logger_config['logfile_path'])
self.logger = Logger.create_logger_by_dict(_logger_config)
self.server_config = server_config
self.app = app
if self.app is None:
self.app = Flask(__name__)
CORS(self.app)
self.app.debug = self.debug
self.app.send_file_max_age_default = datetime.timedelta(
seconds=1) # 设置文件缓存1秒
self.app.config['JSON_AS_ASCII'] = False # 显示中文
# 上传文件大小限制
self.app.config['MAX_CONTENT_LENGTH'] = math.floor(
self.server_config['max_upload_size'] * 1024 * 1024)
# 插件字典，先定义清单，启动前完成加载
self.extend_plugin_path = self.server_config.get(
'extend_plugin_path', '')
self.plugins = dict()
# 装载数据管理模块
self.qa_manager = QAManager(
self.server_config['answerdb'],
self.server_config['milvus'],
self.server_config['bert_client'],
logger=self.logger,
excel_batch_num=self.server_config['excel_batch_num'],
excel_engine=self.server_config['excel_engine'])
# 装载NLP
_nlp_config = self.server_config['nlp_config']
_user_dict = None
if _nlp_config['user_dict'] != '':
_user_dict = _nlp_config['user_dict']
if _user_dict.startswith('.'):
# 相对路径
_user_dict = os.path.join(self.execute_path, _user_dict)
_set_dictionary = None
if _nlp_config['set_dictionary'] != '':
_set_dictionary = _nlp_config['set_dictionary']
if _set_dictionary.startswith('.'):
# 相对路径
_set_dictionary = os.path.join(self.execute_path,
_set_dictionary)
self.nlp = NLP(plugins=self.plugins,
data_manager_para=self.qa_manager.DATA_MANAGER_PARA,
set_dictionary=None if _nlp_config['set_dictionary']
== '' else _nlp_config['set_dictionary'],
user_dict=_user_dict,
enable_paddle=_nlp_config['enable_paddle'],
parallel_num=_nlp_config.get('parallel_num', None),
logger=self.logger)
# 初始化QA模块
self.qa = QA(self.qa_manager,
self.nlp,
self.server_config['execute_path'],
plugins=self.plugins,
qa_config=self.server_config['qa_config'],
redis_config=self.server_config['redis'],
logger=self.logger)
# 动态加载路由
self.api_class = [Qa, QaDataManager]
# 完成插件的加载
# plugins函数字典，格式为{'type':{'class_name': {'fun_name': fun, }, },}
self.load_plugins(os.path.join(self.execute_path, 'plugins'))
if self.extend_plugin_path != '':
if self.extend_plugin_path[0:1] == '.':
# 相对路径
self.extend_plugin_path = os.path.join(self.execute_path,
self.extend_plugin_path)
self.load_plugins(self.extend_plugin_path)
# 安全关联
_security = self.server_config['security']
self.token_serializer = Serializer(
_security['secret_key'],
_security['token_expire'],
salt=bytes(_security['salt'], encoding='utf-8'),
algorithm_name=_security['algorithm_name'])
# 验证ip白名单处理
_security['token_server_auth_ip_list'] = _security[
'token_server_auth_ip_list'].split(',')
# 增加令牌服务的路由
if _security['enable_token_server']:
self.api_class.append(TokenServer)
# 增加静态路径
_static_path = self.server_config['static_path']
if _static_path[0:1] == '.':
# 相对路径
_static_path = os.path.realpath(
os.path.join(self.execute_path, _static_path))
self.app.static_folder = os.path.join(_static_path, 'static')
self.app.static_url_path = '/static/'
# 增加客户端路由
if self.server_config['enable_client']:
# 客户端路由api服务
self.api_class.append(Client)
# 创建测试用户
if self.server_config['add_test_login_user']:
_user = RestfulApiUser.get_or_none(
RestfulApiUser.user_name == 'test')
if _user is None:
self.register_user('test', '123456')
# 加入客户端主页
self.app.url_map.add(Rule('/', endpoint='client', methods=['GET']))
self.app.view_functions['client'] = self._client_view_function
FlaskTool.add_route_by_class(self.app, self.api_class)
self._log_debug(str(self.app.url_map))
def __init__(self, server_config: dict, app: Flask = None, **kwargs):
"""
以图搜图服务初始化
@param {dict} server_config - 服务配置字典
@param {Flask} app=None - 服务
"""
self.kwargs = kwargs
self.debug = server_config.get('debug', True)
self.execute_path = server_config['execute_path']
RunTool.set_global_var('EXECUTE_PATH', self.execute_path)
# 日志处理
self.logger: Logger = None
if 'logger' in server_config.keys():
_logger_config = server_config['logger']
if len(_logger_config['conf_file_name']
) > 0 and _logger_config['conf_file_name'][0] == '.':
# 相对路径
_logger_config['conf_file_name'] = os.path.join(
self.execute_path, _logger_config['conf_file_name'])
if len(_logger_config['logfile_path']
) > 0 and _logger_config['logfile_path'][0] == '.':
# 相对路径
_logger_config['logfile_path'] = os.path.join(
self.execute_path, _logger_config['logfile_path'])
self.logger = Logger.create_logger_by_dict(_logger_config)
# 加载管道配置
RunTool.set_global_var('PIPELINE_PROCESSER_PARA',
server_config['pipeline']['processer_para'])
RunTool.set_global_var('PIPELINE_ROUTER_PARA',
server_config['pipeline']['router_para'])
_plugins_path_list = server_config['pipeline']['plugins_path'].split(
',')
for _plugins_path in _plugins_path_list:
Pipeline.load_plugins_by_path(
os.path.join(self.execute_path, _plugins_path.strip()))
self.server_config = server_config
self.app = app
if self.app is None:
self.app = Flask(__name__)
CORS(self.app)
self.app.debug = self.debug
self.app.send_file_max_age_default = datetime.timedelta(
seconds=1) # 设置文件缓存1秒
self.app.config['JSON_AS_ASCII'] = False # 显示中文
# 上传文件大小限制
self.app.config['MAX_CONTENT_LENGTH'] = math.floor(
self.server_config['max_upload_size'] * 1024 * 1024)
# 装载搜索引擎服务
self.search_engine = SearchEngine(self.server_config,
logger=self.logger)
# 动态加载路由
self.api_class = [
SearchServer,
]
# 增加客户端demo访问
if self.server_config['enable_client']:
# 增加静态路径
_static_path = self.server_config['static_path']
if _static_path[0:1] == '.':
# 相对路径
_static_path = os.path.realpath(
os.path.join(self.execute_path, _static_path))
self.app.static_folder = _static_path
self.app.static_url_path = 'static'
# 加入客户端主页
self.app.url_map.add(Rule('/', endpoint='client', methods=['GET']))
self.app.view_functions['client'] = self._client_view_function
FlaskTool.add_route_by_class(self.app, self.api_class)
self._log_debug(str(self.app.url_map))
}
},
"root": {
"level": "DEBUG",
"handlers": []
}
}
""",
'auto_create_conf': False,
'is_create_logfile_by_day': False,
'call_fun_level': 0
}
# 日志对象
LOGGER = Logger.create_logger_by_dict(_logger_config)
# SocketIO处理函数
def server_on_connect():
"""
客户端连接
"""
print('server: client connect!')
def server_on_disconnect():
print('server: client disconnect!')
def sever_on_add(data):
def __init__(self, config_dict):
"""
初始化构造函数
@param {dict} server_config_dict - 服务器的初始化参数字典（console节点）
"""
# 初始化参数
self._config_dict = copy.deepcopy(config_dict)
# 将部分公共参数放到全局变量
self._console_global_para = RunTool.get_global_var('CONSOLE_GLOBAL_PARA')
self._console_global_para['name'] = self._config_dict['name']
self._console_global_para['version'] = self._config_dict['version']
self._console_global_para['shell_cmd_name'] = self._config_dict['shell_cmd_name']
self._console_global_para['language'] = self._config_dict['language']
self._console_global_para['shell_encoding'] = self._config_dict['shell_encoding'] # 控制台编码
self._console_global_para['exit_with_prompt'] = self._config_dict.get(
'exit_with_prompt', 'y') # 是否提示退出
# i18n多语言加载
_trans_file_path = None
if self._config_dict['i18n'] == '':
_trans_file_path = os.path.join(
self._console_global_para['execute_file_path'], 'i18n/'
)
else:
_trans_file_path = self._config_dict['i18n']
_i18n_obj: SimpleI18N = get_global_i18n()
if _i18n_obj is None:
_i18n_obj = SimpleI18N(
lang=self._config_dict['language'],
trans_file_path=_trans_file_path,
trans_file_prefix='message',
auto_loads=True
)
else:
# 装载默认的多国语言
_i18n_obj.load_trans_from_dir(
_trans_file_path, 'message', encoding='utf-8', append=True
)
_i18n_obj.lang = self._config_dict['language']
# 再装载自身命令的国际语言文件
_i18n_obj.load_trans_from_dir(
os.path.join(os.path.realpath(FileTool.get_file_path(__file__)), 'i18n/'),
'message', encoding='utf-8', append=True
)
set_global_i18n(_i18n_obj)
# 装载默认执行函数
_temp_dict = self._config_dict['default_dealfun']
self._default_cmd_dealfun = self._import_and_init_class(
_temp_dict['module_name'],
_temp_dict['class_name'],
_temp_dict['extend_path'],
_temp_dict['init_para'],
as_name='' if 'as_name' not in _temp_dict.keys() else _temp_dict['as_name']
).cmd_dealfun
_temp_dict = self._config_dict['on_abort']
self._on_abort = self._import_and_init_class(
_temp_dict['module_name'],
_temp_dict['class_name'],
_temp_dict['extend_path'],
_temp_dict['init_para'],
as_name='' if 'as_name' not in _temp_dict.keys() else _temp_dict['as_name']
).cmd_dealfun
_temp_dict = self._config_dict['on_exit']
self._on_exit = self._import_and_init_class(
_temp_dict['module_name'],
_temp_dict['class_name'],
_temp_dict['extend_path'],
_temp_dict['init_para'],
as_name='' if 'as_name' not in _temp_dict.keys() else _temp_dict['as_name']
).cmd_dealfun
# 遍历参数装载
self._is_fw_help_command = False
self._init_cmd_paras()
# 加入到CONSOLE_GLOBAL_PARA参数中
self._console_global_para['CMD_PARA'] = self._CMD_PARA
self._console_global_para['CMD_HELP_INFO'] = self._CMD_HELP_INFO
self._console_global_para['CMD_LIST'] = self._CMD_LIST
# 检查help命令是否框架自带的，如果是则增加提示帮助
if self._is_fw_help_command:
self._CMD_PARA['help']['word_para'] = self._CMD_LIST
# 控制台启动时的提示语言
self._CONSOLE_TIPS = StringTool.json_to_object(self._config_dict['start_tips'])
# 日志对象
_logger = None
if 'logger' in self._config_dict.keys():
_logger = Logger.create_logger_by_dict(
self._config_dict['logger'])
# 颜色调整
_color_set = self._config_dict.get('color_set', None)
if _color_set is not None:
_input_color = _color_set.get('input', None)
if _input_color is not None:
_color_set[''] = _input_color
# 初始化命令行工具对象
self._prompt = PromptPlus(
message=self._config_dict['message'],
default='', # 默认输入值
cmd_para=self._CMD_PARA, # 命令定义参数
default_dealfun=self._default_cmd_dealfun, # 默认处理函数
on_abort=self._on_abort, # Ctrl + C 取消本次输入执行函数
on_exit=self._on_exit, # Ctrl + D 关闭命令行执行函数
logger=_logger, # 日志
color_set=_color_set, # 命令行配色方案
)
# 最后才添加这个对象
self._console_global_para['prompt_obj'] = self._prompt # 将命令行工具对象放到可访问的参数中
# 执行命令处理类的后初始化函数
for _key in self._import_object_dict.keys():
if hasattr(self._import_object_dict[_key], 'init_after_console_init'):
self._import_object_dict[_key].init_after_console_init()
_truncate = (_opts.get('truncate', 'false') == 'true') # 是否清空标志，与操作搭配使用
_milvus = _opts.get('del_milvus', None) # 要删除的问题分类清单，用,分隔
_db = (_opts.get('del_db', 'false') == 'true') # 要重置数据库
# 获取配置文件信息
_execute_path = os.path.realpath(FileTool.get_file_path(__file__))
if _config is None:
_config = os.path.join(_execute_path, 'conf/server.xml')
_config_xml = SimpleXml(_config, encoding=_encoding)
_server_config = _config_xml.to_dict()['server']
# 日志对象
_logger: Logger = None
if 'logger' in _server_config.keys():
_logger = Logger.create_logger_by_dict(_server_config['logger'])
# 连接数据库操作对象
_qa_manager = QAManager(_server_config['answerdb'],
_server_config['milvus'],
_server_config['bert_client'],
logger=_logger,
excel_batch_num=_server_config['excel_batch_num'],
excel_engine=_server_config['excel_engine'],
load_para=False)
# 执行操作
if _import is not None:
# 导入excel文件
_qa_manager.import_questions_by_xls(_import, reset_questions=_truncate)
elif _milvus is not None: