def init_data(self): try: project_base_path = ph.get_local_project_path(os.path.dirname(os.path.abspath(__file__)), 1) range_standard_dir = os.path.join(project_base_path, 'data', 'range_items') for file in os.listdir(range_standard_dir): file_path = os.path.join(range_standard_dir, file) if os.path.isfile(file_path): datas = list() with open(file_path, 'r', encoding='utf-8') as range_file: filename = os.path.basename(file_path) range_datas = range_file.read().split('\n') for range_data in range_datas: range_data = range_data.split('##') if len(range_data) < 3: continue _sri = Sri.StdRangeItem() _sri.range_code = range_data[0] _sri.range_version = filename[:-4] _sri.std_code = range_data[1] _sri.std_name = range_data[2] datas.append(_sri) Eh().batch_index_data(datas, cm.get_es_index_name(self.path_config), cm.get_es_doc_type(self.path_config)) except Exception as e: log_helper.log_out('error', 'File: ' + e.__traceback__.tb_frame.f_globals['__file__'] + ', lineon: ' + str(e.__traceback__.tb_lineno) + ', error info: ' + str(e)) raise
def es_init(): try: project_base_path = ph.get_local_project_path( os.path.dirname(os.path.abspath(__file__)), 1) path_config = os.path.join(project_base_path, 'config/normal.yaml') host = cm.get_es_host(path_config) port = cm.get_es_port(path_config) index_name = cm.get_es_index_name(path_config) index_type = cm.get_es_doc_type(path_config) es_obj = Eh(index_name, index_type, user_name=None, password=None, port=port, ip=host) log_helper.log_out('info', 'start Retrieve initialization') # 创建索引结构 es_obj.create_index(sri.StdRangeItem()) datas = list() # 业务数据来源 business_data = data_sources() for ele in business_data: _sri = sri.StdRangeItem() _sri.std_name = ele datas.append(_sri) # 批量插入数据 es_obj.batch_index_data(datas) log_helper.log_out('info', 'Retrieve initialization Success') except Exception as e: log_helper.log_out( 'error', 'File: ' + e.__traceback__.tb_frame.f_globals['__file__'] + ', lineon: ' + str(e.__traceback__.tb_lineno) + ', error info: ' + str(e))
def query_range_list(): data_list = list() base_path = ph.get_local_project_path( os.path.abspath(os.path.dirname(__file__)), 2) with open(os.path.join(base_path, 'rangeMatching/config', 'model_range_mapping.txt'), 'r', encoding='utf-8') as ps: range_types = ps.read().split('\n') for range_type in range_types: range_type = range_type.split('##') if len(range_type) != 4: continue data_dict = dict() data_dict['znname'] = range_type[0] data_dict['modelcode'] = range_type[1] data_dict['rangecode'] = range_type[2] data_dict['modellevel'] = range_type[3] data_list.append(data_dict) return data_list
def es_search(search_text): project_base_path = ph.get_local_project_path( os.path.dirname(os.path.abspath(__file__)), 1) path_config = os.path.join(project_base_path, 'config/normal.yaml') host = cm.get_es_host(path_config) port = cm.get_es_port(path_config) index_name = cm.get_es_index_name(path_config) index_type = cm.get_es_doc_type(path_config) es_obj = Eh(index_name, index_type, user_name=None, password=None, port=port, ip=host) _sri = sri.StdRangeItem() filters = list() from elasticsearch_dsl import Q # filters.append(Q('term', range_code=range_code)) # filters.append(Q('term', range_version=range_version)) matchings = list() matchings.append(Q('match', std_name=search_text)) search_res = es_obj.search_data(_sri, filters, matchings) return search_res
def __init__(self): project_base_path = ph.get_local_project_path(os.path.dirname(os.path.abspath(__file__)), 3) self.path_config = os.path.join(project_base_path, 'config/normal.yaml')
from elasticsearch_dsl import Document, Keyword, Text from commonUtil import yaml_helper import os from commonUtil import path_helper as ph project_base_path = ph.get_local_project_path(os.path.dirname(os.path.abspath(__file__)), 2) path_config = yaml_helper.get_data_from_yaml( os.path.join(project_base_path, 'config/normal.yaml')) class StdRangeItem(Document): # range_code = Keyword() # range_version = Keyword() # range_type = Keyword() std_code = Keyword() std_name = Text(analyzer='snowball') class Index: name = path_config['index']['range_matching']['index_name'] # class Meta: # index = path_config['index']['range_matching']['index_name'] # doc_type = path_config['index']['range_matching']['doc_type']
import logging import time import os from logging.handlers import RotatingFileHandler from commonUtil import path_helper as ph from commonUtil import yaml_helper project_address = ph.get_local_project_path( os.path.dirname(os.path.abspath(__file__)), 1) business_path_config = yaml_helper.get_data_from_yaml(project_address + '/config/normal.yaml') local_business_logs_path = business_path_config[ 'range_match_local_business_logs_path'] print_console_flag = business_path_config['print_console_flag'] log_file_backup_count = business_path_config['log_file_backup_count'] log_file_limit_size = business_path_config['log_file_limit_size'] # param:日志级别、日志路径、日志内容、是否打印在控制台 def log_out(log_level, log_content): rq_ymd = time.strftime(u'%Y-%m-%d', time.localtime(time.time())) if not os.path.exists(local_business_logs_path + '/' + rq_ymd): os.makedirs(local_business_logs_path + '/' + rq_ymd) logger = logging.getLogger() formatter = logging.Formatter( u"%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s" ) if log_level == 'debug': logger.setLevel(logging.DEBUG) debug_log_name = local_business_logs_path + '/' + rq_ymd + '/' + u'debug.log' # maxBytes:最大10M, backupCount:备份数量 fh_debug = RotatingFileHandler(debug_log_name,