def get_init_objs(cls) -> dict: """ 获取工具初始化对象 @returns {dict} - 初始化对象 """ # 获取配置文件信息 _execute_path = os.path.realpath(FileTool.get_file_path(__file__)) _config = os.path.join(_execute_path, '../conf/server.xml') _config_xml = SimpleXml(_config, encoding='utf-8') _server_config = _config_xml.to_dict()['server'] # 日志对象 _logger: Logger = None if 'logger' in _server_config.keys(): _logger = Logger.create_logger_by_dict(_server_config['logger']) # 连接数据库操作对象 _qa_manager = QAManager( _server_config['answerdb'], _server_config['milvus'], _server_config['bert_client'], logger=_logger, excel_batch_num=_server_config['excel_batch_num'], excel_engine=_server_config['excel_engine'], load_para=False) return {'logger': _logger, 'qa_manager': _qa_manager}
def init_pipeline_plugins(): """ 装载管道插件 """ # 装载配置 _execute_path = os.path.realpath(os.path.join( os.path.dirname(__file__), os.path.pardir, 'search_by_image' )) RunTool.set_global_var('EXECUTE_PATH', _execute_path) _config = os.path.join(_execute_path, 'conf/server_jade.xml') _config_xml = SimpleXml(_config, encoding='utf-8') _server_config = _config_xml.to_dict()['server'] RunTool.set_global_var( 'PIPELINE_PROCESSER_PARA', _server_config['pipeline']['processer_para'] ) RunTool.set_global_var('PIPELINE_ROUTER_PARA', _server_config['pipeline']['router_para']) _plugins_path_list = _server_config['pipeline']['plugins_path'].split(',') for _plugins_path in _plugins_path_list: Pipeline.load_plugins_by_path( os.path.join(_execute_path, _plugins_path.strip()) ) _logger: Logger = None if 'logger' in _server_config.keys(): _logger_config = _server_config['logger'] if len(_logger_config['conf_file_name']) > 0 and _logger_config['conf_file_name'][0] == '.': # 相对路径 _logger_config['conf_file_name'] = os.path.join( _execute_path, _logger_config['conf_file_name'] ) if len(_logger_config['logfile_path']) > 0 and _logger_config['logfile_path'][0] == '.': # 相对路径 _logger_config['logfile_path'] = os.path.join( _execute_path, _logger_config['logfile_path'] ) _logger = Logger.create_logger_by_dict(_logger_config) # 创建空管道用于测试 _empty_pipeline = Pipeline('empty', '{}', logger=_logger) RunTool.set_global_var('EMPTY_PIPELINE', _empty_pipeline) # 创建测试管道 _jade_pipeline = Pipeline( 'jade_search', _server_config['pipeline']['pipeline_config']['JadeSearch'], logger=_logger ) RunTool.set_global_var('JADE_PIPELINE', _jade_pipeline)
def __init__(self): """ 初始化测试参数 """ # 获取配置 self.execute_path = os.path.realpath( os.path.join(os.path.dirname(__file__), os.path.pardir, 'search_by_image')) RunTool.set_global_var('EXECUTE_PATH', self.execute_path) _config = os.path.join(self.execute_path, 'conf/server.xml') _config_xml = SimpleXml(_config, encoding='utf-8') self.server_config = _config_xml.to_dict()['server'] # 装载管道插件 RunTool.set_global_var( 'PIPELINE_PROCESSER_PARA', self.server_config['pipeline']['processer_para']) RunTool.set_global_var('PIPELINE_ROUTER_PARA', self.server_config['pipeline']['router_para']) _plugins_path_list = self.server_config['pipeline'][ 'plugins_path'].split(',') for _plugins_path in _plugins_path_list: Pipeline.load_plugins_by_path( os.path.join(self.execute_path, _plugins_path.strip())) # 日志对象 self.logger: Logger = None if 'logger' in self.server_config.keys(): _logger_config = self.server_config['logger'] if len(_logger_config['conf_file_name'] ) > 0 and _logger_config['conf_file_name'][0] == '.': # 相对路径 _logger_config['conf_file_name'] = os.path.join( self.execute_path, _logger_config['conf_file_name']) if len(_logger_config['logfile_path'] ) > 0 and _logger_config['logfile_path'][0] == '.': # 相对路径 _logger_config['logfile_path'] = os.path.join( self.execute_path, _logger_config['logfile_path']) self.logger = Logger.create_logger_by_dict(_logger_config) # 创建搜索引擎 self.search_engine = SearchEngine(self.server_config, logger=self.logger)
def test_xml_to_dict(self): """ 测试xml转换为字典的情况 """ print('测试SimpleXml - to_dict') _file = self.file_path + '/to_dict.xml' _pfile = SimpleXml(_file, obj_type=EnumXmlObjType.File, encoding=None, use_chardet=True, remove_blank_text=True) _item_dict_xpaths = {'/data/country[2]/list': None} _dict = _pfile.to_dict(item_dict_xpaths=_item_dict_xpaths) print(_dict) self.assertTrue( _dict['data'][0][0] == 2, '失败:测试SimpleXml - to_dict - 检查1:%d' % _dict['data'][0][0]) self.assertTrue( _dict['data'][0][3][3] is True, '失败:测试SimpleXml - to_dict - 检查2:%s' % str(_dict['data'][0][3][3])) self.assertTrue( _dict['data'][1]['list'][0]['b1'] == 'b1', '失败:测试SimpleXml - to_dict - 检查3:%s' % str(_dict['data'][1]['list'][0]['b1']))
def console_main(execute_file_path=None, default_config_file=None, console_self_config: dict = None, **kwargs): """ 启动命令行框架的主函数 @param {string} execute_file_path=None - 外部调用该函数应将程序主目录传入,这样才能找到配置文件 @param {string} default_config_file=None - 如果您希望用不同的配置文件路径作为默认路径,可传入该函数指定 @param {dict} console_self_config=None - 命令行自定义配置信息,将添加到全局参数中 """ # 获取命令行参数,需要外部传入config、encoding参数 # 例如 console.py config=/conf/config.xml encoding=utf-8 help=y shell_cmd=以命令行方式执行命令 shell_cmdfile=cmdfile.txt cmdfile_encoding=utf-8 CONSOLE_GLOBAL_PARA = RunTool.get_global_var('CONSOLE_GLOBAL_PARA') if CONSOLE_GLOBAL_PARA is None: CONSOLE_GLOBAL_PARA = {} RunTool.set_global_var('CONSOLE_GLOBAL_PARA', CONSOLE_GLOBAL_PARA) # 自定义配置信息 CONSOLE_GLOBAL_PARA['console_self_config'] = console_self_config # 程序主目录 if execute_file_path is None: CONSOLE_GLOBAL_PARA['execute_file_path'] = os.path.realpath( FileTool.get_file_path(__file__)) else: CONSOLE_GLOBAL_PARA['execute_file_path'] = execute_file_path # 工作目录,可以通过cd命令切换,通过pwd命令查看工作目录路径 CONSOLE_GLOBAL_PARA['work_path'] = os.getcwd() _cmd_opts = RunTool.get_kv_opts() _default_config_file = default_config_file if _default_config_file is None: _default_config_file = os.path.join( CONSOLE_GLOBAL_PARA['execute_file_path'], 'conf/config.xml') _config = ( _default_config_file ) if 'config' not in _cmd_opts.keys() else _cmd_opts['config'] _encoding = 'utf-8' if 'encoding' not in _cmd_opts.keys() else _cmd_opts['encoding'] CONSOLE_GLOBAL_PARA['config_encoding'] = _encoding CONSOLE_GLOBAL_PARA['config_file'] = _config # 获取配置文件信息 _config_xml = SimpleXml(os.path.realpath(_config), encoding=_encoding) _config_dict = _config_xml.to_dict() # 启动控制台服务 _server = ConsoleServer(_config_dict['console']) # 判断是否 if 'help' in _cmd_opts.keys(): # 执行帮助命令 _lang = _config_dict['console']['language'] _help_tips = StringTool.json_to_object( _config_dict['console']['shell_cmd_help'] ) _tips = '' # 如果找不到对应的语言,优先找英语,如果再找不到就找第一个 if _lang in _help_tips.keys(): _tips = _help_tips[_lang] elif 'en' in _help_tips.keys(): _tips = _help_tips['en'] else: _tips = _help_tips[_help_tips.keys()[0]] _print_str = '\r\n'.join(_tips).replace( '{{VERSION}}', _config_dict['console']['version'] ).replace( '{{NAME}}', _config_dict['console']['name'] ).replace( '{{SHELL_CMD_NAME}}', _config_dict['console']['shell_cmd_name'] ) # 打印 print(_print_str) elif 'shell_cmd' in _cmd_opts.keys(): # 命令行模式执行 _cmd_list = None if _cmd_opts['shell_cmd'][0: 1] == '[' and _cmd_opts['shell_cmd'][-1:] == ']': # 是json数组格式 _cmd_list = StringTool.json_to_object(_cmd_opts['shell_cmd']) else: _cmd_list = [_cmd_opts['shell_cmd']] # 逐个命令执行 for _cmd in _cmd_list: _result = _server.call_cmd_directly(_cmd, shell_cmd=True) if not _result.is_success(): # 执行有错误,不继续执行 exit(1) # 正常完成 exit(0) elif 'shell_cmdfile' in _cmd_opts.keys(): _file_encoding = None if 'cmdfile_encoding' in _cmd_opts.keys() and _cmd_opts['cmdfile_encoding'] != '': _file_encoding = _cmd_opts['cmdfile_encoding'] _cmd_text = FileTool.get_file_text(_cmd_opts['shell_cmdfile'], encoding=_file_encoding) _cmd_text = _cmd_text.replace('\r\n', '\n').replace('\r', '\n') _cmd_list = _cmd_text.split('\n') # 逐个命令执行 for _cmd in _cmd_list: _result = _server.call_cmd_directly(_cmd, shell_cmd=True) if not _result.is_success(): # 执行有错误,不继续执行 exit(1) # 正常完成 exit(0) else: _server.start_console()
def _graph_cmd_dealfun(self, message='', cmd='', cmd_para='', prompt_obj=None, **kwargs): """ 生成APP网络关系图 @param {string} message='' - prompt提示信息 @param {string} cmd - 执行的命令key值 @param {string} cmd_para - 传入的命令参数(命令后的字符串,去掉第一个空格) @param {PromptPlus} prompt_obj=None - 传入调用函数的PromptPlus对象,可以通过该对象的一些方法控制输出显示 @param {kwargs} - 传入的主进程的初始化kwargs对象 @returns {CResult} - 命令执行结果,可通过返回错误码10101通知框架退出命令行, 同时也可以通过CResult对象的 print_str属性要求框架进行打印处理 """ _ok_result = CResult(code='00000') try: # 获取参数及处理参数 _execute_path = self._console_global_para['execute_file_path'] _run_para = { 'file': '', 'formatter': 'col-list', 'comment': 'graph', 'outformat': 'pdf', 'direction': 'forward', 'trace_id': '', 'display_no_trace': 'false', 'down_flow_first': 'true', 'trace_relations': None, 'save': '', 'temp': os.path.join(_execute_path, 'temp'), 'view': 'true', 'cleanup': 'false', 'graph_config': os.path.join(_execute_path, 'conf/graph_config.xml'), 'template': 'default' } _in_para = self._cmd_para_to_dict(cmd_para, name_with_sign=False) _run_para.update(_in_para) if '{para}1' not in _run_para.keys(): prompt_obj.prompt_print(_('you must give the $1 para!', 'file')) return CResult(code='20999') _run_para['file'] = _run_para['{para}1'] if _run_para['save'] == '': _run_para['save'] = 'graph.pdf' # 样式模板信息处理 _style_xml = SimpleXml(_run_para['graph_config'], encoding='utf-8') _style_dict = _style_xml.to_dict()['graphviz']['templates'].get( _run_para['template'], { 'graph': {}, 'node': {}, 'edge': {}, 'edge_color': {}, 'trace': {} } ) # 关联关系线颜色 _name_mapping = _style_dict['edge_color'].get('name_mapping', {}) _use_round_color = _style_dict['edge_color'].get('use_round_color', False) _round_color = _style_dict['edge_color'].get( 'round_color', '').replace(' ', '').split(',') # 生成关系字典 _graph_json = self._formatter_col_list(_run_para['file'], prompt_obj=prompt_obj) # 处理生成调用链的情况 _display_no_trace = _run_para['display_no_trace'] == 'true' if _run_para['trace_id'] != '': # 参与追踪的关系名 if _run_para['trace_relations'] is None: # 全部关系 _run_para['trace_relations'] = _graph_json['relation_name'] else: _run_para['trace_relations'] = _run_para['trace_relations'].split(',') # 获取追踪信息字典 _trace_info = self._get_trace_info( _graph_json, _run_para['trace_id'], _run_para['down_flow_first'] == 'true', _run_para['direction'], _run_para['trace_relations'] ) else: _trace_info = { 'trace_id': '', 'up_nodes': list(), 'down_nodes': list(), 'up_edges': list(), 'down_edges': list() } # 调用链的颜色配置 _attr_center_node = _style_dict.get('trace', {}).get('center_node', {}) _attr_up_node = _style_dict.get('trace', {}).get('up_node', {}) _attr_down_node = _style_dict.get('trace', {}).get('down_node', {}) _attr_up_edge = _style_dict.get('trace', {}).get('up_edge', {}) _attr_down_edge = _style_dict.get('trace', {}).get('down_edge', {}) # 处理关系线条的颜色映射 _relation_color = dict() _index = 0 for _relation_name in _graph_json['relation_name']: if _relation_name in _name_mapping: _relation_color[_relation_name] = _name_mapping[_relation_name] elif _use_round_color: _color_index = _index % len(_round_color) _relation_color[_relation_name] = _round_color[_color_index] _index += 1 else: _relation_color[_relation_name] = '' # 开始处理画图 _dot = Digraph( comment=_run_para['comment'], format=_run_para['outformat'], encoding='utf-8', directory=_run_para['temp'], graph_attr=_style_dict['graph'], node_attr=_style_dict['node'], edge_attr=_style_dict['edge'] ) # 开始画节点 _added_rank = list() # 已添加的分组 for _id, _info in _graph_json['list'].items(): if _info['rank'] != '': # 有分组处理 if _info['rank'] not in _added_rank: _added_rank.append(_info['rank']) with _dot.subgraph() as _sub_dot: _sub_dot.attr(rank='same') for _sub_id in _graph_json['rank'][_info['rank']]: # 节点颜色设置 if _sub_id == _trace_info['trace_id']: # 中心节点 _attr = _attr_center_node elif _sub_id in _trace_info['up_nodes']: _attr = _attr_up_node elif _sub_id in _trace_info['down_nodes']: _attr = _attr_down_node elif _display_no_trace and _trace_info['trace_id'] != '': # 不在链中且不应显示 continue else: _attr = {} _sub_dot.node( _sub_id, label=_graph_json['list'][_sub_id]['name'], **_attr ) else: # 节点颜色设置 if _id == _trace_info['trace_id']: # 中心节点 _attr = _attr_center_node elif _id in _trace_info['up_nodes']: _attr = _attr_up_node elif _id in _trace_info['down_nodes']: _attr = _attr_down_node elif _display_no_trace and _trace_info['trace_id'] != '': # 不在链中且不应显示 continue else: _attr = {} _dot.node(_id, label=_info['name'], **_attr) # 开始画关联线 for _id, _info in _graph_json['list'].items(): for _key, _list in _info.items(): if _key in ('name', 'rank'): continue # 判断颜色 _edge_color = _relation_color[_key] for _temp_id in _list: if _temp_id == '': continue if _run_para['direction'] == 'reverse': _head_id = _temp_id _tail_id = _id else: _head_id = _id _tail_id = _temp_id # 处理颜色 _findstr = '%s&&%s' % (_head_id, _tail_id) if _findstr in _trace_info['up_edges']: _attr = _attr_up_edge elif _findstr in _trace_info['down_edges']: _attr = _attr_down_edge elif _display_no_trace and _trace_info['trace_id'] != '': continue else: _attr = {'color': _edge_color} _dot.edge(_head_id, _tail_id, **_attr) # 保存图片 _dir, _filename = os.path.split(os.path.abspath(_run_para['save'])) _filename = FileTool.get_file_name_no_ext(_filename) _dot.render( filename=_filename, directory=_dir, format=_run_para['outformat'], view=(_run_para['view'] == 'true'), cleanup=(_run_para['cleanup'] == 'true') ) except Exception as e: _prin_str = '%s (%s):\n%s' % ( _('execution exception'), str(e), traceback.format_exc() ) prompt_obj.prompt_print(_prin_str) return CResult(code='20999') # 结束 return _ok_result
from HiveNetLib.base_tools.file_tool import FileTool from HiveNetLib.simple_xml import SimpleXml # 根据当前文件路径将包路径纳入,在非安装的情况下可以引用到 sys.path.append( os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) from chat_robot.lib.data_manager import QAManager from chat_robot.lib.qa import QA from chat_robot.lib.loader import QAServerLoader # 数据管理模块 _file_path = os.path.realpath(FileTool.get_file_path(__file__)) _execute_path = os.path.join(_file_path, os.path.pardir, 'chat_robot') # _config = os.path.join(_execute_path, os.path.pardir, 'chat_robot/conf/server.xml') _config = os.path.join(_execute_path, './conf/server.xml') _config_xml = SimpleXml(_config, encoding='utf-8') SERVER_CONFIG = _config_xml.to_dict()['server'] SERVER_CONFIG['debug'] = True SERVER_CONFIG['config'] = _config SERVER_CONFIG['encoding'] = 'utf-8' SERVER_CONFIG['execute_path'] = _execute_path # 装载服务 _loader = QAServerLoader(SERVER_CONFIG) _qa_manager: QAManager = _loader.qa_manager # 导入Excel数据 # _qa_manager.import_questions_by_xls( # os.path.join(_execute_path, '../test/questions.xlsx'), reset_questions=True # )
# 获取配置信息值 _import = _opts.get('import', None) # 要导入的excel文件 _config = _opts.get('config', None) # 指定配置文件 _encoding = _opts.get('encoding', 'utf-8') # 配置文件编码 _truncate = (_opts.get('truncate', 'false') == 'true') # 是否清空标志,与操作搭配使用 _milvus = _opts.get('del_milvus', None) # 要删除的问题分类清单,用,分隔 _db = (_opts.get('del_db', 'false') == 'true') # 要重置数据库 # 获取配置文件信息 _execute_path = os.path.realpath(FileTool.get_file_path(__file__)) if _config is None: _config = os.path.join(_execute_path, 'conf/server.xml') _config_xml = SimpleXml(_config, encoding=_encoding) _server_config = _config_xml.to_dict()['server'] # 日志对象 _logger: Logger = None if 'logger' in _server_config.keys(): _logger = Logger.create_logger_by_dict(_server_config['logger']) # 连接数据库操作对象 _qa_manager = QAManager(_server_config['answerdb'], _server_config['milvus'], _server_config['bert_client'], logger=_logger, excel_batch_num=_server_config['excel_batch_num'], excel_engine=_server_config['excel_engine'], load_para=False)