def index(self, index_name, body, params=None, id=None, **kwargs):
"""
Create or update a document
:param index_name:
:param body:
:param params:
:param id: if None, will generate, if not None, will replace index if existed
:param kwargs:
:return:
"""
if not self.indextool().exists(index_name):
raise ValueError('index not existed: {}'.format(index_name))
if params:
body = DocTools.render(body, params)
# print(body)
doctype = IndexTools.mapping_get_doctype(
self.indextool().get_mapping(index_name))
if id:
return self._es.index(index=index_name,
body=body,
doc_type=doctype,
id=id,
**kwargs)
else:
return self._es.index(index=index_name,
body=body,
doc_type=doctype,
**kwargs)
def indextool(self):
"""
Get indextool instance
:return:
"""
if not self._indextool:
self._indextool = IndexTools.from_es(self._es)
return self._indextool
def exists(self, index_name, id, **kwargs):
"""
Check if a document exists in an index or not
:param index_name:
:param id:
:param kwargs:
:return: boolean
"""
if not self.indextool().exists(index_name):
raise ValueError('index not existed: {}'.format(index_name))
doctype = IndexTools.mapping_get_doctype(
self.indextool().get_mapping(index_name))
return self._es.exists(index=index_name,
id=id,
doc_type=doctype,
**kwargs)
def delete(self, index_name, id, **kwargs):
"""
Delete a document with id = `id` in an index
:param index_name:
:param id:
:param kwargs:
:return:
"""
if not self.indextool().exists(index_name):
raise ValueError('index not existed: {}'.format(index_name))
doctype = IndexTools.mapping_get_doctype(
self.indextool().get_mapping(index_name))
return self._es.delete(index=index_name,
id=id,
doc_type=doctype,
**kwargs)
def bulk(self,
index_name,
actions,
doctype=None,
thread_count=1,
check_index_existed=True,
**kwargs):
"""
Do bulk actions, if thread_count = 1, otherwise call parallel_bulk
:param index_name:
:param actions: any iterable, can also be a generator, in search result format (with `_source`) or orignal format
:param thread_count: 1 if using bulk, other wise, usi aarop
:param kwargs:
:return:
"""
if check_index_existed:
if not self.indextool().exists(index_name):
raise ValueError('index not existed: {}'.format(index_name))
if not doctype:
doctype = IndexTools.mapping_get_doctype(
self.indextool().get_mapping(index_name))
if thread_count <= 1:
print('Normal bulk')
return elasticsearch.helpers.bulk(self._es,
actions,
index=index_name,
doc_type=doctype,
**kwargs)
else:
print('Parallel bulk', thread_count)
return deque(elasticsearch.helpers.parallel_bulk(
self._es,
actions,
index=index_name,
doc_type=doctype,
thread_count=thread_count,
**kwargs),
maxlen=0)
def get(self, index_name, id, source=False, **kwargs):
"""
Get a document in an index by it id
:param index_name:
:param id:
:param source:
:param kwargs:
:return:
"""
if not self.indextool().exists(index_name):
raise ValueError('index not existed: {}'.format(index_name))
doctype = IndexTools.mapping_get_doctype(
self.indextool().get_mapping(index_name))
if source:
return self._es.get_source(index=index_name,
id=id,
doc_type=doctype,
**kwargs)
else:
return self._es.get(index=index_name,
id=id,
doc_type=doctype,
**kwargs)
from elastictools.indextools import IndexTools
es = IndexTools.from_url('http://localhost:9200')