def init_data(self):
try:
project_base_path = ph.get_local_project_path(os.path.dirname(os.path.abspath(__file__)), 1)
range_standard_dir = os.path.join(project_base_path, 'data', 'range_items')
for file in os.listdir(range_standard_dir):
file_path = os.path.join(range_standard_dir, file)
if os.path.isfile(file_path):
datas = list()
with open(file_path, 'r', encoding='utf-8') as range_file:
filename = os.path.basename(file_path)
range_datas = range_file.read().split('\n')
for range_data in range_datas:
range_data = range_data.split('##')
if len(range_data) < 3:
continue
_sri = Sri.StdRangeItem()
_sri.range_code = range_data[0]
_sri.range_version = filename[:-4]
_sri.std_code = range_data[1]
_sri.std_name = range_data[2]
datas.append(_sri)
Eh().batch_index_data(datas, cm.get_es_index_name(self.path_config),
cm.get_es_doc_type(self.path_config))
except Exception as e:
log_helper.log_out('error', 'File: ' + e.__traceback__.tb_frame.f_globals['__file__']
+ ', lineon: ' + str(e.__traceback__.tb_lineno) + ', error info: ' + str(e))
raise
def es_init():
try:
project_base_path = ph.get_local_project_path(
os.path.dirname(os.path.abspath(__file__)), 1)
path_config = os.path.join(project_base_path, 'config/normal.yaml')
host = cm.get_es_host(path_config)
port = cm.get_es_port(path_config)
index_name = cm.get_es_index_name(path_config)
index_type = cm.get_es_doc_type(path_config)
es_obj = Eh(index_name,
index_type,
user_name=None,
password=None,
port=port,
ip=host)
log_helper.log_out('info', 'start Retrieve initialization')
# 创建索引结构
es_obj.create_index(sri.StdRangeItem())
datas = list()
# 业务数据来源
business_data = data_sources()
for ele in business_data:
_sri = sri.StdRangeItem()
_sri.std_name = ele
datas.append(_sri)
# 批量插入数据
es_obj.batch_index_data(datas)
log_helper.log_out('info', 'Retrieve initialization Success')
except Exception as e:
log_helper.log_out(
'error', 'File: ' +
e.__traceback__.tb_frame.f_globals['__file__'] + ', lineon: ' +
str(e.__traceback__.tb_lineno) + ', error info: ' + str(e))
def query_range_list():
data_list = list()
base_path = ph.get_local_project_path(
os.path.abspath(os.path.dirname(__file__)), 2)
with open(os.path.join(base_path, 'rangeMatching/config', 'model_range_mapping.txt'), 'r', encoding='utf-8') as ps:
range_types = ps.read().split('\n')
for range_type in range_types:
range_type = range_type.split('##')
if len(range_type) != 4:
continue
data_dict = dict()
data_dict['znname'] = range_type[0]
data_dict['modelcode'] = range_type[1]
data_dict['rangecode'] = range_type[2]
data_dict['modellevel'] = range_type[3]
data_list.append(data_dict)
return data_list
def es_search(search_text):
project_base_path = ph.get_local_project_path(
os.path.dirname(os.path.abspath(__file__)), 1)
path_config = os.path.join(project_base_path, 'config/normal.yaml')
host = cm.get_es_host(path_config)
port = cm.get_es_port(path_config)
index_name = cm.get_es_index_name(path_config)
index_type = cm.get_es_doc_type(path_config)
es_obj = Eh(index_name,
index_type,
user_name=None,
password=None,
port=port,
ip=host)
_sri = sri.StdRangeItem()
filters = list()
from elasticsearch_dsl import Q
# filters.append(Q('term', range_code=range_code))
# filters.append(Q('term', range_version=range_version))
matchings = list()
matchings.append(Q('match', std_name=search_text))
search_res = es_obj.search_data(_sri, filters, matchings)
return search_res
def __init__(self):
project_base_path = ph.get_local_project_path(os.path.dirname(os.path.abspath(__file__)), 3)
self.path_config = os.path.join(project_base_path, 'config/normal.yaml')
from elasticsearch_dsl import Document, Keyword, Text
from commonUtil import yaml_helper
import os
from commonUtil import path_helper as ph
project_base_path = ph.get_local_project_path(os.path.dirname(os.path.abspath(__file__)), 2)
path_config = yaml_helper.get_data_from_yaml(
os.path.join(project_base_path, 'config/normal.yaml'))
class StdRangeItem(Document):
# range_code = Keyword()
# range_version = Keyword()
# range_type = Keyword()
std_code = Keyword()
std_name = Text(analyzer='snowball')
class Index:
name = path_config['index']['range_matching']['index_name']
# class Meta:
# index = path_config['index']['range_matching']['index_name']
# doc_type = path_config['index']['range_matching']['doc_type']
import logging
import time
import os
from logging.handlers import RotatingFileHandler
from commonUtil import path_helper as ph
from commonUtil import yaml_helper
project_address = ph.get_local_project_path(
os.path.dirname(os.path.abspath(__file__)), 1)
business_path_config = yaml_helper.get_data_from_yaml(project_address +
'/config/normal.yaml')
local_business_logs_path = business_path_config[
'range_match_local_business_logs_path']
print_console_flag = business_path_config['print_console_flag']
log_file_backup_count = business_path_config['log_file_backup_count']
log_file_limit_size = business_path_config['log_file_limit_size']
# param:日志级别、日志路径、日志内容、是否打印在控制台
def log_out(log_level, log_content):
rq_ymd = time.strftime(u'%Y-%m-%d', time.localtime(time.time()))
if not os.path.exists(local_business_logs_path + '/' + rq_ymd):
os.makedirs(local_business_logs_path + '/' + rq_ymd)
logger = logging.getLogger()
formatter = logging.Formatter(
u"%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s"
)
if log_level == 'debug':
logger.setLevel(logging.DEBUG)
debug_log_name = local_business_logs_path + '/' + rq_ymd + '/' + u'debug.log'
# maxBytes：最大10M， backupCount：备份数量
fh_debug = RotatingFileHandler(debug_log_name,