class EsClientConnection:
host = ''
errorMessage = ''
def __init__(self, host, index=None, type=None, body=None):
'''
创建的时候需要两个都要存在
:param host:
:param index:
:param type:
:param body:
'''
self.host = host
self.conn = Elasticsearch([self.host])
# 初始化mapping设置,即创建index
indexExists = self.conn.indices.exists(index=index)
typeExists = self.conn.indices.exists_type(index=index, doc_type=type)
if body is not None:
if indexExists is not True:
if typeExists is not True:
self.conn.indices.create(index=index, body=body)
else:
self.errorMessage = 'index not exists and type exists. it is not possible!'
else:
if typeExists is not True:
self.errorMessage = 'index index exists and type not exists'
else:
self.errorMessage = 'index exists and type exists. you not need create it'
def __del__(self):
self.close()
def check(self):
'''
输出当前系统的ES信息
:return:
'''
return self.conn.info()
def insertDocument(self, index, type, body, id=None):
'''
插入一条数据body到指定的index、指定的type下;可指定Id,若不指定,ES会自动生成
:param index: 待插入的index值
:param type: 待插入的type值
:param body: 待插入的数据 -> dict型
:param id: 自定义Id值
:return:
'''
return self.conn.index(index=index, doc_type=type, body=body, id=id)
def insertDataFrame(self, index, type, dataFrame):
'''
批量插入接口;
bulk接口所要求的数据列表结构为:[{{optionType}: {Condition}}, {data}]
其中optionType可为index、delete、update
Condition可设置每条数据所对应的index值和type值
data为具体要插入/更新的单条数据
:param index: 默认插入的index值
:param type: 默认插入的type值
:param dataFrame: 待插入数据集
:return:
'''
dataList = dataFrame.to_dict(orient='records')
insertHeadInfoList = [{"index": {}} for i in range(len(dataList))]
temp = [dict] * (len(dataList) * 2)
temp[::2] = insertHeadInfoList
temp[1::2] = dataList
try:
return self.conn.bulk(index=index, doc_type=type, body=temp)
except Exception as e:
return str(e)
def deleteDocById(self, index, type, id):
'''
删除指定index、type、id对应的数据
:param index:
:param type:
:param id:
:return:
'''
return self.conn.delete(index=index, doc_type=type, id=id)
def deleteDocByQuery(self, index, query, type=None):
'''
删除idnex下符合条件query的所有数据
:param index:
:param query: 满足DSL语法格式
:param type:
:return:
'''
return self.conn.delete_by_query(index=index,
body=query,
doc_type=type)
def deleteAllDocByIndex(self, index, type=None):
'''
删除指定index下的所有数据
:param index:
:return:
'''
try:
query = {'query': {'match_all': {}}}
return self.conn.delete_by_query(index=index,
body=query,
doc_type=type)
except Exception as e:
return str(e) + ' -> ' + index
def searchDoc(self, index=None, type=None, body=None):
'''
查找index下所有符合条件的数据
:param index:
:param type:
:param body: 筛选语句,符合DSL语法格式
:return:
'''
return self.conn.search(index=index, doc_type=type, body=body)
def getDocById(self, index, type, id):
'''
获取指定index、type、id对应的数据
:param index:
:param type:
:param id:
:return:
'''
return self.conn.get(index=index, doc_type=type, id=id)
def updateDocById(self, index, type, id, body=None):
'''
更新指定index、type、id所对应的数据
:param index:
:param type:
:param id:
:param body: 待更新的值
:return:
'''
return self.conn.update(index=index, doc_type=type, id=id, body=body)
def close(self):
if self.conn is not None:
try:
self.conn.close()
except Exception as e:
pass
finally:
self.conn = None
def mysqlToEs(self, mysqlData):
doc = []
for value in mysqlData:
doc.append({"index": {}})
doc.append(value)
self.conn.bulk(index='product', doc_type='tour_product', body=doc)
from elasticsearch5 import Elasticsearch
import chardet
import pickle
es = Elasticsearch([
'https://search-el-dev-znz7hdtpcgghjcq4vatwtc3xiu.ap-northeast-2.es.amazonaws.com:443'
])
print(es.info())
page = es.search(index='slowquery-2018.01.09',
doc_type='elltdev',
body={'query': {
'match_all': {}
}})
pickle.dump(page, open('es01.pkl', 'wb'))
#f = open("es01.json",'w')
#data = str(page)
# print(chardet.detect(page))
# f.write(data)
# f.close()
from elasticsearch5 import Elasticsearch
useIndex = 'tw_user_database_*'
TWEETSINDEX = "tweets_database*"
# host = "192.168.209.113"
# port = "9200"
host = "192.168。8.200"
port = "9201"
es_client = Elasticsearch([{"host": host, "port": port}])
info = es_client.info()
userid = "25073877"
body = {"query": {"match": {"user.id": userid}}}
rs = es_client.search(index=TWEETSINDEX, body=body)
print rs
print type(rs)
