def _get_ids_worker(args):
from utils.es import ESIndexer
from pyes import MatchAllQuery
es_kwargs, start, step = args
q = MatchAllQuery().search()
q.sort = [{'entrezgene': 'asc'}, {'ensembl.gene': 'asc'}]
q.fields = []
q.start = start
q.size = step
esi = ESIndexer(**es_kwargs)
cnt = esi.count()['count']
res = esi.conn.search_raw(q)
assert res['hits']['total'] == cnt
return [doc['_id'] for doc in res['hits']['hits']]
def _get_results(self, index=0, count=None):
"""
@returns: elasticsearch iterator over results
defined by self.query
"""
query = self.db_query
if self.db_query.is_empty():
query = MatchAllQuery()
if self._ordering:
query.sort = self._ordering
query = Search(query)
query.bulk_read = 5000
if index > 0 and count is not None:
query.start = index
query.size = count
# else:
# query = Search(query, start=0, size=5000)
#print "query", self.query.tables, query
return self._connection.search(query, indices=[self.connection.db_name], doc_types=self.query.model._meta.db_table)