def test_GeoDistanceFilter(self):
gq = GeoDistanceFilter("pin.location", {"lat": 40, "lon": 70}, "200km")
q = FilteredQuery(MatchAllQuery(), gq)
resultset = self.conn.search(query=q, indices=["test-mindex"])
self.assertEquals(resultset.total, 1)
gq = GeoDistanceFilter("pin.location", [70, 40], "200km")
q = FilteredQuery(MatchAllQuery(), gq)
resultset = self.conn.search(query=q, indices=["test-mindex"])
self.assertEquals(resultset.total, 1)
def test_remove(self):
"""Ensure we can properly delete from ElasticSearch via DocManager.
"""
docc = {'_id': '1', 'name': 'John', 'ns': 'test.test'}
self.elastic_doc.upsert(docc)
self.elastic_doc.commit()
res = self.elastic_conn.search(MatchAllQuery())
self.assertTrue(len(res) == 1)
self.elastic_doc.remove(docc)
self.elastic_doc.commit()
res = self.elastic_conn.search(MatchAllQuery())
self.assertTrue(len(res) == 0)
def test_remove(self):
"""Ensure we can properly delete from ElasticSearch via DocManager.
"""
docc = {'_id': '1', 'name': 'John', 'ns': 'test.test'}
ElasticDoc.upsert(docc)
ElasticDoc.commit()
res = elastic.search(MatchAllQuery())
self.assertTrue(len(res) == 1)
ElasticDoc.remove(docc)
ElasticDoc.commit()
res = elastic.search(MatchAllQuery())
self.assertTrue(len(res) == 0)
print("PASSED REMOVE")
def get_last_doc(self):
"""Returns the last document stored in the Elastic engine.
"""
result = self.elastic.search(MatchAllQuery(), size=1, sort='_ts:desc')
for item in result:
return item
def __call__(self, dquery):
filters = []
catalog = self.catalogtool._catalog
idxs = catalog.indexes.keys()
query = MatchAllQuery()
for key, value in dquery.items():
if key not in idxs:
continue
index = getIndex(catalog, key)
if index is None:
continue
qq = index.get_query(key, value)
if qq is None:
continue
if type(qq) == tuple:
qq, is_query = qq
else:
is_query = False
if is_query:
query = qq
else:
filters.append(qq)
if len(filters) == 0:
return query
else:
return FilteredQuery(query, ANDFilter(filters))
def count(self, limit=None):
query = self.db_query
if self.db_query.is_empty():
query = MatchAllQuery()
res = self._connection.count(query,
doc_types=self.query.model._meta.db_table)
return res["count"]
def test_upsert(self):
"""Ensure we can properly insert into ElasticSearch via DocManager.
"""
docc = {'_id': '1', 'name': 'John', 'ns': 'test.test'}
ElasticDoc.upsert(docc)
ElasticDoc.commit()
res = elastic.search(MatchAllQuery())
for doc in res:
self.assertTrue(doc['_id'] == '1' and doc['name'] == 'John')
docc = {'_id': '1', 'name': 'Paul', 'ns': 'test.test'}
ElasticDoc.upsert(docc)
ElasticDoc.commit()
res = elastic.search(MatchAllQuery())
for doc in res:
self.assertTrue(doc['_id'] == '1' and doc['name'] == 'Paul')
print("PASSED UPSERT")
def test_GeoBoundingBoxFilter(self):
gq = GeoBoundingBoxFilter("pin.location",
location_tl={
"lat": 40.717,
"lon": 70.99
},
location_br={
"lat": 40.03,
"lon": 72.0
})
q = FilteredQuery(MatchAllQuery(), gq)
resultset = self.conn.search(query=q, indices=["test-mindex"])
self.assertEquals(resultset.total, 1)
gq = GeoBoundingBoxFilter("pin.location", [70.99, 40.717],
[74.1, 40.03])
q = FilteredQuery(MatchAllQuery(), gq)
result2 = self.conn.search(query=q, indices=["test-mindex"])
self.assertEquals(result2.total, 1)
def test_upsert(self):
"""Ensure we can properly insert into ElasticSearch via DocManager.
"""
docc = {'_id': '1', 'name': 'John', 'ns': 'test.test'}
self.elastic_doc.upsert(docc)
self.elastic_doc.commit()
res = self.elastic_conn.search(MatchAllQuery())
for doc in res:
self.assertTrue(doc['_id'] == '1' and doc['name'] == 'John')
def test_ReconvertDoubles(self):
"""Regression test for issue#6.
Pyes used to fail when getting a query respones in which a document
contained a list of doubles.
"""
q = MatchAllQuery()
result = self.conn.search(query=q, indexes=["test-pindex"])
self.assertEquals(result['hits']['total'], 2)
def test_GeoPolygonFilter(self):
gq = GeoPolygonFilter("pin.location", [{
"lat": 50,
"lon": -30
}, {
"lat": 30,
"lon": -80
}, {
"lat": 80,
"lon": -90
}])
q = FilteredQuery(MatchAllQuery(), gq)
resultset = self.conn.search(query=q, indices=["test-mindex"])
self.assertEquals(resultset.total, 1)
gq = GeoPolygonFilter("pin.location",
[[-30, 50], [-80, 30], [-90, 80]])
q = FilteredQuery(MatchAllQuery(), gq)
resultset = self.conn.search(query=q, indices=["test-mindex"])
self.assertEquals(resultset.total, 1)
def get_last_doc(self):
"""Returns the last document stored in the Elastic engine.
"""
it = None
q = MatchAllQuery()
result = self.elastic.search(q, size=1, sort={'_ts:desc'})
for it in result:
r = it
break
return r
def _get_results(self):
"""
@returns: elasticsearch iterator over results
defined by self.query
"""
query = self.db_query
if self.db_query.is_empty():
query = MatchAllQuery()
if self._ordering:
query.sort = self._ordering
#print "query", self.query.tables, query
return self._connection.search(
query,
indices=[self.connection.db_name],
doc_types=self.query.model._meta.db_table)
def test_full_search(self):
"""Query ElasticSearch for all docs via API and via DocManager's
_search(), compare.
"""
docc = {'_id': '1', 'name': 'John', 'ns': 'test.test'}
self.elastic_doc.upsert(docc)
docc = {'_id': '2', 'name': 'Paul', 'ns': 'test.test'}
self.elastic_doc.upsert(docc)
self.elastic_doc.commit()
search = self.elastic_doc._search()
search2 = self.elastic_conn.search(MatchAllQuery())
self.assertTrue(len(search) == len(search2))
self.assertTrue(len(search) != 0)
for i in range(0, len(search)):
self.assertTrue(list(search)[i] == list(search2)[i])
def afterCompletion(self, transaction):
tdata = get()
if not tdata.registered:
return
es = tdata.es
if es.mode == DISABLE_MODE:
tdata.reset()
return
success = transaction.status == Status.COMMITTED
query = FilteredQuery(MatchAllQuery(),
TermFilter('transaction_id', tdata.tid))
conn = es.conn
# NEED to refresh here otherwise we'll have inconsistencies
conn.refresh()
try:
docs = conn.search(query, es.catalogsid, es.trns_catalogtype,
sort='order:desc')
docs.count() # force executing
except ElasticSearchException:
# XXX uh oh, nasty, we have a problem. Let's log it.
warn("Error trying to abort transaction: %s" %(
traceback.format_exc()))
tdata.reset()
return
for doc in docs:
conn.delete(es.catalogsid, es.trns_catalogtype, doc.get_id())
if not success:
if doc.action == Actions.add:
# if it was an add action, remove delete
conn.delete(es.catalogsid, es.catalogtype, doc.uid)
elif doc.action in (Actions.modify, Actions.delete):
# if it was a modify or delete, restore the doc
restored_doc = loads(doc.data)
conn.index(restored_doc, es.catalogsid, es.catalogtype, doc.uid)
# NEED to refresh here otherwise we'll have inconsistencies
conn.refresh()
tdata.reset()
def get_pubs(filter=MissingFilter('types')):
q = FilteredQuery(MatchAllQuery(), filter)
pubs = conn.search(query=q, indices=e_index, doc_types="immo")
return pubs
def _search(self):
"""For test purposes only. Performs search on Elastic with empty query.
Does not have to be implemented.
"""
results = self.elastic.search(MatchAllQuery())
return results
ftool = FileTools()
ftrans = FormatTranslator()
# 1. Create Connection
conn = ES()
# 2. Index Data
dataset_json = open("../dataset.json")
dataset = json.load(dataset_json)['data']
for data in dataset:
conn.index(data, "example_index", "example_type",
"example_id_" + str(dataset.index(data)))
# 3. Create Simple Query
query = MatchAllQuery()
# 4. Create Simple Aggregation
agg = TermsAgg('agg1', field="name", sub_aggs=[], size=100)
# 5. Get Result
search = Search(query, size=5)
search.agg.add(agg)
print search.serialize()
result = conn.search(search, "example_index", "example_type")
for i in result:
print json.dumps(i, indent=2)
print json.dumps(result.aggs, indent=2)
def getAllElasticsTransactions(self):
return self.es.conn.search(MatchAllQuery(), self.es.catalogsid,
self.es.trns_catalogtype)
# test
from mediaresearchapp.tasks import MediaAggregateSQLTask
if __name__ == '__main__':
es = ES("127.0.0.1:9200", default_indices='mediaaggregate')
# Filters
filters = [GeoDistanceFilter('location', [40.0, 9.00], 20, 'arc', 'km')]
# filters = [TermFilter('message', 'elastic'),
# GeoDistanceFilter('locations',
# {"lat": 40.0, "lon": 9.00},
# 20, 'arc', 'km')
# ]
filter = ANDFilter(filters)
q = FilteredQuery(MatchAllQuery(), filter)
results = es.search(q)
for r in results:
print r
break
q4 = RegexTermQuery('city', 'bang.*')
print q4
resultset = es.search(q4)
for r in resultset:
print r
query_str = {
"query": {
"termquery": [{
"fieldname1": "value"