def get_init_objs(cls) -> dict:
"""
获取工具初始化对象
@returns {dict} - 初始化对象
"""
# 获取配置文件信息
_execute_path = os.path.realpath(FileTool.get_file_path(__file__))
_config = os.path.join(_execute_path, '../conf/server.xml')
_config_xml = SimpleXml(_config, encoding='utf-8')
_server_config = _config_xml.to_dict()['server']
# 日志对象
_logger: Logger = None
if 'logger' in _server_config.keys():
_logger = Logger.create_logger_by_dict(_server_config['logger'])
# 连接数据库操作对象
_qa_manager = QAManager(
_server_config['answerdb'],
_server_config['milvus'],
_server_config['bert_client'],
logger=_logger,
excel_batch_num=_server_config['excel_batch_num'],
excel_engine=_server_config['excel_engine'],
load_para=False)
return {'logger': _logger, 'qa_manager': _qa_manager}
def init_pipeline_plugins():
"""
装载管道插件
"""
# 装载配置
_execute_path = os.path.realpath(os.path.join(
os.path.dirname(__file__), os.path.pardir, 'search_by_image'
))
RunTool.set_global_var('EXECUTE_PATH', _execute_path)
_config = os.path.join(_execute_path, 'conf/server_jade.xml')
_config_xml = SimpleXml(_config, encoding='utf-8')
_server_config = _config_xml.to_dict()['server']
RunTool.set_global_var(
'PIPELINE_PROCESSER_PARA', _server_config['pipeline']['processer_para']
)
RunTool.set_global_var('PIPELINE_ROUTER_PARA', _server_config['pipeline']['router_para'])
_plugins_path_list = _server_config['pipeline']['plugins_path'].split(',')
for _plugins_path in _plugins_path_list:
Pipeline.load_plugins_by_path(
os.path.join(_execute_path, _plugins_path.strip())
)
_logger: Logger = None
if 'logger' in _server_config.keys():
_logger_config = _server_config['logger']
if len(_logger_config['conf_file_name']) > 0 and _logger_config['conf_file_name'][0] == '.':
# 相对路径
_logger_config['conf_file_name'] = os.path.join(
_execute_path, _logger_config['conf_file_name']
)
if len(_logger_config['logfile_path']) > 0 and _logger_config['logfile_path'][0] == '.':
# 相对路径
_logger_config['logfile_path'] = os.path.join(
_execute_path, _logger_config['logfile_path']
)
_logger = Logger.create_logger_by_dict(_logger_config)
# 创建空管道用于测试
_empty_pipeline = Pipeline('empty', '{}', logger=_logger)
RunTool.set_global_var('EMPTY_PIPELINE', _empty_pipeline)
# 创建测试管道
_jade_pipeline = Pipeline(
'jade_search',
_server_config['pipeline']['pipeline_config']['JadeSearch'],
logger=_logger
)
RunTool.set_global_var('JADE_PIPELINE', _jade_pipeline)
def __init__(self):
"""
初始化测试参数
"""
# 获取配置
self.execute_path = os.path.realpath(
os.path.join(os.path.dirname(__file__), os.path.pardir,
'search_by_image'))
RunTool.set_global_var('EXECUTE_PATH', self.execute_path)
_config = os.path.join(self.execute_path, 'conf/server.xml')
_config_xml = SimpleXml(_config, encoding='utf-8')
self.server_config = _config_xml.to_dict()['server']
# 装载管道插件
RunTool.set_global_var(
'PIPELINE_PROCESSER_PARA',
self.server_config['pipeline']['processer_para'])
RunTool.set_global_var('PIPELINE_ROUTER_PARA',
self.server_config['pipeline']['router_para'])
_plugins_path_list = self.server_config['pipeline'][
'plugins_path'].split(',')
for _plugins_path in _plugins_path_list:
Pipeline.load_plugins_by_path(
os.path.join(self.execute_path, _plugins_path.strip()))
# 日志对象
self.logger: Logger = None
if 'logger' in self.server_config.keys():
_logger_config = self.server_config['logger']
if len(_logger_config['conf_file_name']
) > 0 and _logger_config['conf_file_name'][0] == '.':
# 相对路径
_logger_config['conf_file_name'] = os.path.join(
self.execute_path, _logger_config['conf_file_name'])
if len(_logger_config['logfile_path']
) > 0 and _logger_config['logfile_path'][0] == '.':
# 相对路径
_logger_config['logfile_path'] = os.path.join(
self.execute_path, _logger_config['logfile_path'])
self.logger = Logger.create_logger_by_dict(_logger_config)
# 创建搜索引擎
self.search_engine = SearchEngine(self.server_config,
logger=self.logger)
def test_xml_to_dict(self):
"""
测试xml转换为字典的情况
"""
print('测试SimpleXml - to_dict')
_file = self.file_path + '/to_dict.xml'
_pfile = SimpleXml(_file,
obj_type=EnumXmlObjType.File,
encoding=None,
use_chardet=True,
remove_blank_text=True)
_item_dict_xpaths = {'/data/country[2]/list': None}
_dict = _pfile.to_dict(item_dict_xpaths=_item_dict_xpaths)
print(_dict)
self.assertTrue(
_dict['data'][0][0] == 2,
'失败：测试SimpleXml - to_dict - 检查1：%d' % _dict['data'][0][0])
self.assertTrue(
_dict['data'][0][3][3] is True,
'失败：测试SimpleXml - to_dict - 检查2：%s' % str(_dict['data'][0][3][3]))
self.assertTrue(
_dict['data'][1]['list'][0]['b1'] == 'b1',
'失败：测试SimpleXml - to_dict - 检查3：%s' %
str(_dict['data'][1]['list'][0]['b1']))
def console_main(execute_file_path=None, default_config_file=None, console_self_config: dict = None, **kwargs):
"""
启动命令行框架的主函数
@param {string} execute_file_path=None - 外部调用该函数应将程序主目录传入，这样才能找到配置文件
@param {string} default_config_file=None - 如果您希望用不同的配置文件路径作为默认路径，可传入该函数指定
@param {dict} console_self_config=None - 命令行自定义配置信息，将添加到全局参数中
"""
# 获取命令行参数，需要外部传入config、encoding参数
# 例如 console.py config=/conf/config.xml encoding=utf-8 help=y shell_cmd=以命令行方式执行命令 shell_cmdfile=cmdfile.txt cmdfile_encoding=utf-8
CONSOLE_GLOBAL_PARA = RunTool.get_global_var('CONSOLE_GLOBAL_PARA')
if CONSOLE_GLOBAL_PARA is None:
CONSOLE_GLOBAL_PARA = {}
RunTool.set_global_var('CONSOLE_GLOBAL_PARA', CONSOLE_GLOBAL_PARA)
# 自定义配置信息
CONSOLE_GLOBAL_PARA['console_self_config'] = console_self_config
# 程序主目录
if execute_file_path is None:
CONSOLE_GLOBAL_PARA['execute_file_path'] = os.path.realpath(
FileTool.get_file_path(__file__))
else:
CONSOLE_GLOBAL_PARA['execute_file_path'] = execute_file_path
# 工作目录，可以通过cd命令切换，通过pwd命令查看工作目录路径
CONSOLE_GLOBAL_PARA['work_path'] = os.getcwd()
_cmd_opts = RunTool.get_kv_opts()
_default_config_file = default_config_file
if _default_config_file is None:
_default_config_file = os.path.join(
CONSOLE_GLOBAL_PARA['execute_file_path'], 'conf/config.xml')
_config = (
_default_config_file
) if 'config' not in _cmd_opts.keys() else _cmd_opts['config']
_encoding = 'utf-8' if 'encoding' not in _cmd_opts.keys() else _cmd_opts['encoding']
CONSOLE_GLOBAL_PARA['config_encoding'] = _encoding
CONSOLE_GLOBAL_PARA['config_file'] = _config
# 获取配置文件信息
_config_xml = SimpleXml(os.path.realpath(_config), encoding=_encoding)
_config_dict = _config_xml.to_dict()
# 启动控制台服务
_server = ConsoleServer(_config_dict['console'])
# 判断是否
if 'help' in _cmd_opts.keys():
# 执行帮助命令
_lang = _config_dict['console']['language']
_help_tips = StringTool.json_to_object(
_config_dict['console']['shell_cmd_help']
)
_tips = ''
# 如果找不到对应的语言，优先找英语，如果再找不到就找第一个
if _lang in _help_tips.keys():
_tips = _help_tips[_lang]
elif 'en' in _help_tips.keys():
_tips = _help_tips['en']
else:
_tips = _help_tips[_help_tips.keys()[0]]
_print_str = '\r\n'.join(_tips).replace(
'{{VERSION}}', _config_dict['console']['version']
).replace(
'{{NAME}}', _config_dict['console']['name']
).replace(
'{{SHELL_CMD_NAME}}', _config_dict['console']['shell_cmd_name']
)
# 打印
print(_print_str)
elif 'shell_cmd' in _cmd_opts.keys():
# 命令行模式执行
_cmd_list = None
if _cmd_opts['shell_cmd'][0: 1] == '[' and _cmd_opts['shell_cmd'][-1:] == ']':
# 是json数组格式
_cmd_list = StringTool.json_to_object(_cmd_opts['shell_cmd'])
else:
_cmd_list = [_cmd_opts['shell_cmd']]
# 逐个命令执行
for _cmd in _cmd_list:
_result = _server.call_cmd_directly(_cmd, shell_cmd=True)
if not _result.is_success():
# 执行有错误，不继续执行
exit(1)
# 正常完成
exit(0)
elif 'shell_cmdfile' in _cmd_opts.keys():
_file_encoding = None
if 'cmdfile_encoding' in _cmd_opts.keys() and _cmd_opts['cmdfile_encoding'] != '':
_file_encoding = _cmd_opts['cmdfile_encoding']
_cmd_text = FileTool.get_file_text(_cmd_opts['shell_cmdfile'], encoding=_file_encoding)
_cmd_text = _cmd_text.replace('\r\n', '\n').replace('\r', '\n')
_cmd_list = _cmd_text.split('\n')
# 逐个命令执行
for _cmd in _cmd_list:
_result = _server.call_cmd_directly(_cmd, shell_cmd=True)
if not _result.is_success():
# 执行有错误，不继续执行
exit(1)
# 正常完成
exit(0)
else:
_server.start_console()
def _graph_cmd_dealfun(self, message='', cmd='', cmd_para='', prompt_obj=None, **kwargs):
"""
生成APP网络关系图
@param {string} message='' - prompt提示信息
@param {string} cmd - 执行的命令key值
@param {string} cmd_para - 传入的命令参数（命令后的字符串，去掉第一个空格）
@param {PromptPlus} prompt_obj=None - 传入调用函数的PromptPlus对象，可以通过该对象的一些方法控制输出显示
@param {kwargs} - 传入的主进程的初始化kwargs对象
@returns {CResult} - 命令执行结果，可通过返回错误码10101通知框架退出命令行, 同时也可以通过CResult对象的
print_str属性要求框架进行打印处理
"""
_ok_result = CResult(code='00000')
try:
# 获取参数及处理参数
_execute_path = self._console_global_para['execute_file_path']
_run_para = {
'file': '',
'formatter': 'col-list',
'comment': 'graph',
'outformat': 'pdf',
'direction': 'forward',
'trace_id': '',
'display_no_trace': 'false',
'down_flow_first': 'true',
'trace_relations': None,
'save': '',
'temp': os.path.join(_execute_path, 'temp'),
'view': 'true',
'cleanup': 'false',
'graph_config': os.path.join(_execute_path, 'conf/graph_config.xml'),
'template': 'default'
}
_in_para = self._cmd_para_to_dict(cmd_para, name_with_sign=False)
_run_para.update(_in_para)
if '{para}1' not in _run_para.keys():
prompt_obj.prompt_print(_('you must give the $1 para!', 'file'))
return CResult(code='20999')
_run_para['file'] = _run_para['{para}1']
if _run_para['save'] == '':
_run_para['save'] = 'graph.pdf'
# 样式模板信息处理
_style_xml = SimpleXml(_run_para['graph_config'], encoding='utf-8')
_style_dict = _style_xml.to_dict()['graphviz']['templates'].get(
_run_para['template'], {
'graph': {},
'node': {},
'edge': {},
'edge_color': {},
'trace': {}
}
)
# 关联关系线颜色
_name_mapping = _style_dict['edge_color'].get('name_mapping', {})
_use_round_color = _style_dict['edge_color'].get('use_round_color', False)
_round_color = _style_dict['edge_color'].get(
'round_color', '').replace(' ', '').split(',')
# 生成关系字典
_graph_json = self._formatter_col_list(_run_para['file'], prompt_obj=prompt_obj)
# 处理生成调用链的情况
_display_no_trace = _run_para['display_no_trace'] == 'true'
if _run_para['trace_id'] != '':
# 参与追踪的关系名
if _run_para['trace_relations'] is None:
# 全部关系
_run_para['trace_relations'] = _graph_json['relation_name']
else:
_run_para['trace_relations'] = _run_para['trace_relations'].split(',')
# 获取追踪信息字典
_trace_info = self._get_trace_info(
_graph_json, _run_para['trace_id'], _run_para['down_flow_first'] == 'true',
_run_para['direction'], _run_para['trace_relations']
)
else:
_trace_info = {
'trace_id': '',
'up_nodes': list(),
'down_nodes': list(),
'up_edges': list(),
'down_edges': list()
}
# 调用链的颜色配置
_attr_center_node = _style_dict.get('trace', {}).get('center_node', {})
_attr_up_node = _style_dict.get('trace', {}).get('up_node', {})
_attr_down_node = _style_dict.get('trace', {}).get('down_node', {})
_attr_up_edge = _style_dict.get('trace', {}).get('up_edge', {})
_attr_down_edge = _style_dict.get('trace', {}).get('down_edge', {})
# 处理关系线条的颜色映射
_relation_color = dict()
_index = 0
for _relation_name in _graph_json['relation_name']:
if _relation_name in _name_mapping:
_relation_color[_relation_name] = _name_mapping[_relation_name]
elif _use_round_color:
_color_index = _index % len(_round_color)
_relation_color[_relation_name] = _round_color[_color_index]
_index += 1
else:
_relation_color[_relation_name] = ''
# 开始处理画图
_dot = Digraph(
comment=_run_para['comment'], format=_run_para['outformat'], encoding='utf-8',
directory=_run_para['temp'],
graph_attr=_style_dict['graph'],
node_attr=_style_dict['node'],
edge_attr=_style_dict['edge']
)
# 开始画节点
_added_rank = list() # 已添加的分组
for _id, _info in _graph_json['list'].items():
if _info['rank'] != '':
# 有分组处理
if _info['rank'] not in _added_rank:
_added_rank.append(_info['rank'])
with _dot.subgraph() as _sub_dot:
_sub_dot.attr(rank='same')
for _sub_id in _graph_json['rank'][_info['rank']]:
# 节点颜色设置
if _sub_id == _trace_info['trace_id']:
# 中心节点
_attr = _attr_center_node
elif _sub_id in _trace_info['up_nodes']:
_attr = _attr_up_node
elif _sub_id in _trace_info['down_nodes']:
_attr = _attr_down_node
elif _display_no_trace and _trace_info['trace_id'] != '':
# 不在链中且不应显示
continue
else:
_attr = {}
_sub_dot.node(
_sub_id, label=_graph_json['list'][_sub_id]['name'], **_attr
)
else:
# 节点颜色设置
if _id == _trace_info['trace_id']:
# 中心节点
_attr = _attr_center_node
elif _id in _trace_info['up_nodes']:
_attr = _attr_up_node
elif _id in _trace_info['down_nodes']:
_attr = _attr_down_node
elif _display_no_trace and _trace_info['trace_id'] != '':
# 不在链中且不应显示
continue
else:
_attr = {}
_dot.node(_id, label=_info['name'], **_attr)
# 开始画关联线
for _id, _info in _graph_json['list'].items():
for _key, _list in _info.items():
if _key in ('name', 'rank'):
continue
# 判断颜色
_edge_color = _relation_color[_key]
for _temp_id in _list:
if _temp_id == '':
continue
if _run_para['direction'] == 'reverse':
_head_id = _temp_id
_tail_id = _id
else:
_head_id = _id
_tail_id = _temp_id
# 处理颜色
_findstr = '%s&&%s' % (_head_id, _tail_id)
if _findstr in _trace_info['up_edges']:
_attr = _attr_up_edge
elif _findstr in _trace_info['down_edges']:
_attr = _attr_down_edge
elif _display_no_trace and _trace_info['trace_id'] != '':
continue
else:
_attr = {'color': _edge_color}
_dot.edge(_head_id, _tail_id, **_attr)
# 保存图片
_dir, _filename = os.path.split(os.path.abspath(_run_para['save']))
_filename = FileTool.get_file_name_no_ext(_filename)
_dot.render(
filename=_filename, directory=_dir,
format=_run_para['outformat'], view=(_run_para['view'] == 'true'),
cleanup=(_run_para['cleanup'] == 'true')
)
except Exception as e:
_prin_str = '%s (%s):\n%s' % (
_('execution exception'), str(e), traceback.format_exc()
)
prompt_obj.prompt_print(_prin_str)
return CResult(code='20999')
# 结束
return _ok_result
from HiveNetLib.base_tools.file_tool import FileTool
from HiveNetLib.simple_xml import SimpleXml
# 根据当前文件路径将包路径纳入，在非安装的情况下可以引用到
sys.path.append(
os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
from chat_robot.lib.data_manager import QAManager
from chat_robot.lib.qa import QA
from chat_robot.lib.loader import QAServerLoader
# 数据管理模块
_file_path = os.path.realpath(FileTool.get_file_path(__file__))
_execute_path = os.path.join(_file_path, os.path.pardir, 'chat_robot')
# _config = os.path.join(_execute_path, os.path.pardir, 'chat_robot/conf/server.xml')
_config = os.path.join(_execute_path, './conf/server.xml')
_config_xml = SimpleXml(_config, encoding='utf-8')
SERVER_CONFIG = _config_xml.to_dict()['server']
SERVER_CONFIG['debug'] = True
SERVER_CONFIG['config'] = _config
SERVER_CONFIG['encoding'] = 'utf-8'
SERVER_CONFIG['execute_path'] = _execute_path
# 装载服务
_loader = QAServerLoader(SERVER_CONFIG)
_qa_manager: QAManager = _loader.qa_manager
# 导入Excel数据
# _qa_manager.import_questions_by_xls(
# os.path.join(_execute_path, '../test/questions.xlsx'), reset_questions=True
# )
# 获取配置信息值
_import = _opts.get('import', None) # 要导入的excel文件
_config = _opts.get('config', None) # 指定配置文件
_encoding = _opts.get('encoding', 'utf-8') # 配置文件编码
_truncate = (_opts.get('truncate', 'false') == 'true') # 是否清空标志，与操作搭配使用
_milvus = _opts.get('del_milvus', None) # 要删除的问题分类清单，用,分隔
_db = (_opts.get('del_db', 'false') == 'true') # 要重置数据库
# 获取配置文件信息
_execute_path = os.path.realpath(FileTool.get_file_path(__file__))
if _config is None:
_config = os.path.join(_execute_path, 'conf/server.xml')
_config_xml = SimpleXml(_config, encoding=_encoding)
_server_config = _config_xml.to_dict()['server']
# 日志对象
_logger: Logger = None
if 'logger' in _server_config.keys():
_logger = Logger.create_logger_by_dict(_server_config['logger'])
# 连接数据库操作对象
_qa_manager = QAManager(_server_config['answerdb'],
_server_config['milvus'],
_server_config['bert_client'],
logger=_logger,
excel_batch_num=_server_config['excel_batch_num'],
excel_engine=_server_config['excel_engine'],
load_para=False)