def batch_index_data(self, cls_obj_arr):
try:
if cls_obj_arr is None:
raise Exception('cls array none')
if len(cls_obj_arr) == 0:
raise Exception('cls array empty')
actions = []
for cls_obj in cls_obj_arr:
action = dict()
action['_index'] = self.index_name
action['_type'] = self.index_type
data = cls_obj.to_dict()
action['_source'] = data
actions.append(action)
success, _ = bulk(self.es,
actions,
index=self.index_name,
raise_on_error=True)
print('Performed %d actions' % success)
except Exception as e:
log_helper.log_out(
'error', 'File: ' +
e.__traceback__.tb_frame.f_globals['__file__'] + ', lineon: ' +
str(e.__traceback__.tb_lineno) + ', error info: ' + str(e))
raise
def init_data(self):
try:
project_base_path = ph.get_local_project_path(os.path.dirname(os.path.abspath(__file__)), 1)
range_standard_dir = os.path.join(project_base_path, 'data', 'range_items')
for file in os.listdir(range_standard_dir):
file_path = os.path.join(range_standard_dir, file)
if os.path.isfile(file_path):
datas = list()
with open(file_path, 'r', encoding='utf-8') as range_file:
filename = os.path.basename(file_path)
range_datas = range_file.read().split('\n')
for range_data in range_datas:
range_data = range_data.split('##')
if len(range_data) < 3:
continue
_sri = Sri.StdRangeItem()
_sri.range_code = range_data[0]
_sri.range_version = filename[:-4]
_sri.std_code = range_data[1]
_sri.std_name = range_data[2]
datas.append(_sri)
Eh().batch_index_data(datas, cm.get_es_index_name(self.path_config),
cm.get_es_doc_type(self.path_config))
except Exception as e:
log_helper.log_out('error', 'File: ' + e.__traceback__.tb_frame.f_globals['__file__']
+ ', lineon: ' + str(e.__traceback__.tb_lineno) + ', error info: ' + str(e))
raise
def __init__(self,
index_name,
index_type,
user_name=None,
password=None,
port=None,
ip="127.0.0.1"):
"""
:param index_name:
:param index_type:
:param user_name:
:param password:
:param port:
:param ip:
"""
try:
self.index_name = index_name
self.index_type = index_type
self.ip = ip
# ÎÞÓû§ÃûÃÜÂë״̬
if not user_name:
self.es = Elasticsearch([ip])
else:
# Óû§ÃûÃÜÂë״̬
self.es = Elasticsearch([ip],
http_auth=(user_name, password),
port=port)
except Exception as e:
log_helper.log_out(
'error', 'File: ' +
e.__traceback__.tb_frame.f_globals['__file__'] + ', lineon: ' +
str(e.__traceback__.tb_lineno) + ', error info: ' + str(e))
raise
def init_structure(self):
try:
_es = Eh()
_es.create_index(Sri.StdRangeItem(), cm.get_es_index_name(self.path_config))
except Exception as e:
log_helper.log_out('error', 'File: ' + e.__traceback__.tb_frame.f_globals['__file__']
+ ', lineon: ' + str(e.__traceback__.tb_lineno) + ', error info: ' + str(e))
raise
def judge_index_existence(self, index_name):
try:
if self.es.indices.exists(index=index_name) is True:
return True
else:
return False
except Exception as e:
log_helper.log_out(
'error', 'File: ' +
e.__traceback__.tb_frame.f_globals['__file__'] + ', lineon: ' +
str(e.__traceback__.tb_lineno) + ', error info: ' + str(e))
raise
def create_index(self, cls_obj):
try:
if cls_obj is None:
raise Exception('_index_mappings none')
if self.judge_index_existence(self.index_name) is True:
self.es.indices.delete(index=self.index_name)
log_helper.log_out('info', 'deleting existing indexes')
cls_obj.init()
except Exception as e:
log_helper.log_out(
'error', 'File: ' +
e.__traceback__.tb_frame.f_globals['__file__'] + ', lineon: ' +
str(e.__traceback__.tb_lineno) + ', error info: ' + str(e))
raise
def delete_document(self, cls, index_name):
try:
if cls is None:
raise Exception('cls none')
if not self.es.indices.exists(index=index_name):
print(index_name + ' dos not exists')
return
res = self.es.indices.delete(index=index_name)
print(res)
except Exception as e:
log_helper.log_out(
'error', 'File: ' +
e.__traceback__.tb_frame.f_globals['__file__'] + ', lineon: ' +
str(e.__traceback__.tb_lineno) + ', error info: ' + str(e))
raise
def search_data(self, cls, filter_qs, match_qs):
try:
if cls is None:
raise Exception('cls none')
s = Search(using=self.es, index=self.index_name)
if filter_qs is not None and len(filter_qs) > 0:
for filter_q in filter_qs:
s = s.filter(filter_q)
if match_qs is not None and len(match_qs) > 0:
for match_q in match_qs:
s = s.query(match_q)
res = s.execute()
return res
except Exception as e:
log_helper.log_out(
'error', 'File: ' +
e.__traceback__.tb_frame.f_globals['__file__'] + ', lineon: ' +
str(e.__traceback__.tb_lineno) + ', error info: ' + str(e))
raise
def es_init():
try:
project_base_path = ph.get_local_project_path(
os.path.dirname(os.path.abspath(__file__)), 1)
path_config = os.path.join(project_base_path, 'config/normal.yaml')
host = cm.get_es_host(path_config)
port = cm.get_es_port(path_config)
index_name = cm.get_es_index_name(path_config)
index_type = cm.get_es_doc_type(path_config)
es_obj = Eh(index_name,
index_type,
user_name=None,
password=None,
port=port,
ip=host)
log_helper.log_out('info', 'start Retrieve initialization')
# 创建索引结构
es_obj.create_index(sri.StdRangeItem())
datas = list()
# 业务数据来源
business_data = data_sources()
for ele in business_data:
_sri = sri.StdRangeItem()
_sri.std_name = ele
datas.append(_sri)
# 批量插入数据
es_obj.batch_index_data(datas)
log_helper.log_out('info', 'Retrieve initialization Success')
except Exception as e:
log_helper.log_out(
'error', 'File: ' +
e.__traceback__.tb_frame.f_globals['__file__'] + ', lineon: ' +
str(e.__traceback__.tb_lineno) + ', error info: ' + str(e))