es.index(doc={
"name": "Bill Clinton",
"parsedtext": """Bill is not
nice guy""",
"uuid": "33333",
"position": 3
},
index=index_name,
doc_type=type_name,
id=3,
bulk=True)
es.force_bulk()
es.update(index=index_name,
doc_type=type_name,
id=2,
script='ctx._source.position += 1')
es.update(index=index_name,
doc_type=type_name,
id=2,
script='ctx._source.position += 1',
bulk=True)
es.delete(index=index_name, doc_type=type_name, id=1, bulk=True)
es.delete(index=index_name, doc_type=type_name, id=3)
es.force_bulk()
es.indices.refresh(index_name)
es.indices.delete_index(index_name)
es = ES()
index_name = "my_index"
type_name = "my_type"
from utils_pyes import create_and_add_mapping
create_and_add_mapping(es, index_name, type_name)
es.index(doc={"name": "Joe Tester", "parsedtext": "Joe Testere nice guy", "uuid": "11111", "position": 1},
index=index_name, doc_type=type_name, id=1)
es.index(doc={"name": "data1", "value": "value1"}, index=index_name, doc_type=type_name + "2", id=1, parent=1)
es.index(doc={"name": "Bill Baloney", "parsedtext": "Bill Testere nice guy", "uuid": "22222", "position": 2},
index=index_name, doc_type=type_name, id=2, bulk=True)
es.index(doc={"name": "data2", "value": "value2"}, index=index_name, doc_type=type_name + "2", id=2, parent=2,
bulk=True)
es.index(doc={"name": "Bill Clinton", "parsedtext": """Bill is not
nice guy""", "uuid": "33333", "position": 3}, index=index_name, doc_type=type_name, id=3, bulk=True)
es.force_bulk()
es.update(index=index_name, doc_type=type_name, id=2, script='ctx._source.position += 1')
es.update(index=index_name, doc_type=type_name, id=2, script='ctx._source.position += 1', bulk=True)
es.delete(index=index_name, doc_type=type_name, id=1, bulk=True)
es.delete(index=index_name, doc_type=type_name, id=3)
es.force_bulk()
es.indices.refresh(index_name)
es.indices.delete_index(index_name)
class DocManager(object):
"""The DocManager class creates a connection to the backend engine and
adds/removes documents, and in the case of rollback, searches for them.
The reason for storing id/doc pairs as opposed to doc's is so that
multiple updates to the same doc reflect the most up to date version as
opposed to multiple, slightly different versions of a doc.
We are using elastic native fields for _id and ns, but we also store
them as fields in the document, due to compatibility issues.
"""
def __init__(self, url, auto_commit=True, unique_key='_id'):
"""Verify Elastic URL and establish a connection.
"""
if verify_url(url) is False:
raise SystemError
self.elastic = ES(server=url)
self.auto_commit = auto_commit
self.doc_type = 'string' # default type is string, change if needed
self.unique_key = unique_key
if auto_commit:
self.run_auto_commit()
def stop(self):
""" Stops the instance
"""
self.auto_commit = False
def upsert(self, doc):
"""Update or insert a document into Elastic
If you'd like to have different types of document in your database,
you can store the doc type as a field in Mongo and set doc_type to
that field. (e.g. doc_type = doc['_type'])
"""
# There is a problem with ES .90.0 and possibly .90.1 with
# indices not be correctly handled.
# This ensures that an upsert correctly happens
doc_type = self.doc_type
index = doc['ns']
doc[self.unique_key] = str(doc[self.unique_key])
doc_id = doc[self.unique_key]
id_query = TextQuery('_id', doc_id)
elastic_cursor = self.elastic.search(query=id_query, indices=index)
if elastic_cursor.total == 0:
self.elastic.index(doc, index, doc_type, doc_id)
else:
self.elastic.update(doc, index, doc_type, doc_id)
self.elastic.refresh()
def remove(self, doc):
"""Removes documents from Elastic
The input is a python dictionary that represents a mongo document.
"""
try:
self.elastic.delete(doc['ns'], 'string', str(doc[self.unique_key]))
except IndexMissingException:
pass
def _remove(self):
"""For test purposes only. Removes all documents in test.test
"""
try:
self.elastic.delete('test.test', 'string', '')
except IndexMissingException:
pass
def search(self, start_ts, end_ts):
"""Called to query Elastic for documents in a time range.
"""
res = ESRange('_ts', from_value=start_ts, to_value=end_ts)
results = self.elastic.search(RangeQuery(res))
return results
def _search(self):
"""For test purposes only. Performs search on Elastic with empty query.
Does not have to be implemented.
"""
results = self.elastic.search(MatchAllQuery())
return results
def commit(self):
"""This function is used to force a refresh/commit.
"""
retry_until_ok(self.elastic.refresh)
def run_auto_commit(self):
"""Periodically commits to the Elastic server.
"""
self.elastic.refresh()
if self.auto_commit:
Timer(1, self.run_auto_commit).start()
def get_last_doc(self):
"""Returns the last document stored in the Elastic engine.
"""
result = self.elastic.search(MatchAllQuery(), size=1, sort='_ts:desc')
for item in result:
return item
