def search(self, query, explain=False, validate=False):
try:
request = ElseParser.parse(query)
except ElseParserException as err:
print(err.pstr)
print(" " * err.loc + "^\n")
print("ERROR:", err)
return 1
params = {}
data_fields = None
if request.query:
data = {'query': {'query_string': {'query': str(request.query), 'default_operator': 'AND'}}}
else:
data = {'query': {'match_all': {}}}
if explain:
data['explain'] = True
if request.filter:
filter = request.filter
if filter.name == 'query':
data['filter'] = {'query': {'query_string': {'query': str(filter), 'default_operator': 'AND'}}}
else:
data['filter'] = {filter.name: {'field': str(filter)}}
if request.facets:
# data['facets'] = {f: {"terms": {"field": f}} for f in request.facets} -- not in python 2.6
data['facets'] = dict((f, {"terms": {"field": f}}) for f in request.facets)
if request.script:
data['script_fields'] = {request.script[0]: {"script": request.script[1]}}
if request.fields:
fields = request.fields
if len(fields) == 1:
if fields[0] == '*':
# all fields
pass
elif fields[0] == 'count(*)':
# TODO: only get count
pass
else:
data_fields = data['fields'] = [fields[0]]
else:
data_fields = data['fields'] = [x for x in fields]
if request.order:
data['sort'] = [{x[0]:x[1]} for x in request.order]
if request.limit:
qfrom = None
qsize = None
if len(request.limit) > 1:
qfrom = request.limit.pop(0)
qsize = request.limit[0]
if qfrom is None:
data['size'] = qsize
elif qfrom >= 0:
data['from'] = qfrom
data['size'] = qsize
else:
#
# limit -1, 1000 => scan request, 1000 items at a time
#
params.update({'search_type': 'scan', 'scroll': '10m', 'size': qsize})
if validate:
command = '/_validate/query'
params.update({'pretty': True, 'explain': True})
# validate doesn't like "query"
if 'query' in data:
q = data.pop('query')
data.update(q)
#
# this is actually {index}/{document-id}/_explain
#
#elif explain:
# command = '/_explain'
# params.update({'pretty': True})
else:
command = '/_search'
command_path = request.index.replace(".", "/") + command
if self.debug:
print()
print("GET", command_path, params or '')
print(" ", pprint.pformat(data))
params.update({'pretty': True})
if self.print_query:
print()
print("; ", _csval(query))
print()
total = None
print_fields = True
do_query = True
while self.es and do_query:
try:
result = self.es.get(command_path, params=params, data=data)
except ConnectionError as err:
print("cannot connect to", self.es.url)
print(err)
return
if self.debug:
print()
print("RESPONSE:", pprint.pformat(result))
print()
if '_scroll_id' in result:
# scan/scroll request
params['scroll_id'] = result['_scroll_id']
if 'search_type' in params:
params.pop('search_type')
command_path = '_search/scroll'
else:
# done
do_query = False
if 'valid' in result:
if 'explanations' in result:
for e in result['explanations']:
print()
for k, v in e.iteritems():
print(k, ':', v)
else:
print("valid:", result['valid'])
return
if 'error' in result:
print("ERROR:", result['error'])
return
if 'shards' in result and 'failures' in result['_shards']:
failures = result['_shards']['failures']
for f in failures:
print("ERROR:", f['reason'])
return
if 'hits' in result:
total = result['hits']['total']
if data_fields:
if print_fields:
print_fields = False
print(_csvline(data_fields))
for _ in result['hits']['hits']:
result_fields = _['fields'] if 'fields' in _ else {}
print(_csvline([_.get(x) or result_fields.get(x) for x in data_fields]))
else:
if result['hits']['hits']:
if print_fields:
print_fields = False
print(_csvline(result['hits']['hits'][0]['_source'].keys()))
else:
do_query = False
for _ in result['hits']['hits']:
print(_csvline([_csval(x) for x in _['_source'].values()]))
if 'facets' in result:
for facet in result['facets']:
print()
print("%s,count" % _csval(facet))
for _ in result['facets'][facet]['terms']:
t = _['term']
c = _['count']
print("%s,%s" % (_csval(t), c))
if do_query and self.debug:
print()
print("GET", command_path, params or '')
print(" ", pprint.pformat(data))
if total is not None:
print()
print("total: ", total)
def search(self, query, explain=False, validate=False):
try:
request = ElseParser.parse(query)
except ElseParserException as err:
print(err.pstr)
print(" " * err.loc + "^\n")
print("ERROR:", err)
return 1
params = {}
data_fields = None
if request.query:
data = {'query': {'query_string': {'query': str(request.query), 'default_operator': 'AND'}}}
else:
data = {'query': {'match_all': {}}}
if explain:
data['explain'] = True
if request.filter:
filter = request.filter
if filter.name == 'query':
data['filter'] = {'query': {'query_string': {'query': str(filter), 'default_operator': 'AND'}}}
else:
data['filter'] = {filter.name: {'field': str(filter)}}
if request.facets:
# data['facets'] = {f: {"terms": {"field": f}} for f in request.facets} -- not in python 2.6
data['facets'] = dict((f, {"terms": {"field": f}}) for f in request.facets)
if request.script:
data['script_fields'] = {request.script[0]: {"script": request.script[1]}}
if request.fields:
fields = request.fields
fields_k = '_source' if self.v5 else 'fields'
if len(fields) == 1:
if fields[0] == '*':
# all fields
pass
elif fields[0] == 'count(*)':
# TODO: only get count
pass
else:
data[fields_k] = [fields[0]]
else:
data[fields_k] = [x for x in fields]
data_fields = data.get(fields_k)
if request.order:
data['sort'] = [{x[0]:x[1]} for x in request.order]
if request.limit:
qfrom = None
qsize = None
if len(request.limit) > 1:
qfrom = request.limit.pop(0)
qsize = request.limit[0]
if qfrom is None:
data['size'] = qsize
elif qfrom >= 0:
data['from'] = qfrom
data['size'] = qsize
else:
#
# limit -1, 1000 => scan request, 1000 items at a time
#
params.update({'search_type': 'scan', 'scroll': '10m', 'size': qsize})
if validate:
command = '/_validate/query'
params.update({'pretty': 'true', 'explain': 'true'})
# validate doesn't like "query"
if 'query' in data:
q = data.pop('query')
data.update(q)
#
# this is actually {index}/{document-id}/_explain
#
# elif explain:
# command = '/_explain'
# params.update({'pretty': 'true'})
else:
command = '/_search'
if request.routing:
command += '?routing=%s' % request.routing
command_path = request.index.replace(".", "/") + command
if self.debug:
HTTPConnection.debuglevel = 1
print()
print("GET", command_path, params or '')
print(" ", pprint.pformat(data))
params.update({'pretty': 'true'})
else:
HTTPConnection.debuglevel = 0
if self.print_query:
print()
print("; ", _csval(query))
print()
total = None
print_fields = True
do_query = True
while self.es and do_query:
try:
result = self.es.get(command_path, params=params, data=data)
except ConnectionError as err:
print("cannot connect to", self.es.url)
print(err)
return
if self.debug:
print()
print("RESPONSE:", pprint.pformat(result))
print()
if '_scroll_id' in result:
# scan/scroll request
params['scroll_id'] = result['_scroll_id']
if 'search_type' in params:
params.pop('search_type')
command_path = '_search/scroll'
else:
# done
do_query = False
if 'valid' in result:
if 'explanations' in result:
for e in result['explanations']:
print()
for k, v in e.iteritems():
print(k, ':', v)
else:
print("valid:", result['valid'])
return
if 'error' in result:
print("ERROR:", result['error'])
return
if 'shards' in result and 'failures' in result['_shards']:
failures = result['_shards']['failures']
for f in failures:
print("ERROR:", f['reason'])
return
if 'hits' in result:
hits = result['hits']
total = hits['total']
if data_fields and not self.v5:
if print_fields:
print_fields = False
print(_csvline(data_fields))
for _ in hits['hits']:
result_fields = _['fields'] if 'fields' in _ else {}
print(_csvline([_.get(x) or result_fields.get(x) for x in data_fields]))
else:
if hits['hits']:
if print_fields:
print_fields = False
print(_csvline(hits['hits'][0]['_source'].keys()))
else:
do_query = False
for _ in hits['hits']:
print(_csvline([_csval(x) for x in _['_source'].values()]))
if 'facets' in result:
for facet in result['facets']:
print()
print("%s,count" % _csval(facet))
for _ in result['facets'][facet]['terms']:
t = _['term']
c = _['count']
print("%s,%s" % (_csval(t), c))
if do_query and self.debug:
print()
print("GET", command_path, params or '')
print(" ", pprint.pformat(data))
if total is not None:
print()
print("total: ", total)
def getESFormatQuery(query, calledFromREST=False):
try:
request = ElseParser.parse(query)
except ElseParserException as err:
print(err.pstr)
print(" " * err.loc + "^\n")
print("ERROR:", err)
if calledFromREST:
return {'status' : 'FAILED','reason' :'%s' %err}
else:
return 1
params = {}
data_fields = None
if request.query:
data = {'query': {'query_string': {'query': str(request.query), 'default_operator': 'AND'}}}
else:
data = {'query': {'match_all': {}}}
#if explain:
# data['explain'] = True
if request.filter:
filter = request.filter
if filter.name == 'query':
data['filter'] = {'query': {'query_string': {'query': str(filter), 'default_operator': 'AND'}}}
else:
data['filter'] = {filter.name: {'field': str(filter)}}
if request.facets:
# data['facets'] = {f: {"terms": {"field": f}} for f in request.facets} -- not in python 2.6
data['facets'] = dict((f, {"terms": {"field": f}}) for f in request.facets)
if request.script:
data['script_fields'] = {request.script[0]: {"script": request.script[1]}}
if request.fields:
fields = request.fields
if len(fields) == 1:
if fields[0] == '*':
# all fields
pass
elif fields[0] == 'count(*)':
# TODO: only get count
pass
else:
data_fields = data['fields'] = [fields[0]]
else:
data_fields = data['fields'] = [x for x in fields]
if request.order:
data['sort'] = [{x[0]:x[1]} for x in request.order]
if request.limit:
qfrom = None
qsize = None
if len(request.limit) > 1:
qfrom = request.limit.pop(0)
qsize = request.limit[0]
if qfrom is None:
data['size'] = qsize
elif qfrom >= 0:
data['from'] = qfrom
data['size'] = qsize
else:
#
# limit -1, 1000 => scan request, 1000 items at a time
#
params.update({'search_type': 'scan', 'scroll': '10m', 'size': qsize})
if calledFromREST :
return {'status' : 'OK','result' : '%s' %data}
else:
return request, data, params
def getESFormatQuery(query, calledFromREST=False):
try:
request = ElseParser.parse(query)
except ElseParserException as err:
print(err.pstr)
print(" " * err.loc + "^\n")
print("ERROR:", err)
if calledFromREST:
return {'status': 'FAILED', 'reason': '%s' % err}
else:
return 1
params = {}
data_fields = None
if request.query:
data = {
'query': {
'query_string': {
'query': str(request.query),
'default_operator': 'AND'
}
}
}
else:
data = {'query': {'match_all': {}}}
#if explain:
# data['explain'] = True
if request.filter:
filter = request.filter
if filter.name == 'query':
data['filter'] = {
'query': {
'query_string': {
'query': str(filter),
'default_operator': 'AND'
}
}
}
else:
data['filter'] = {filter.name: {'field': str(filter)}}
if request.facets:
# data['facets'] = {f: {"terms": {"field": f}} for f in request.facets} -- not in python 2.6
data['facets'] = dict((f, {
"terms": {
"field": f
}
}) for f in request.facets)
if request.script:
data['script_fields'] = {
request.script[0]: {
"script": request.script[1]
}
}
if request.fields:
fields = request.fields
if len(fields) == 1:
if fields[0] == '*':
# all fields
pass
elif fields[0] == 'count(*)':
# TODO: only get count
pass
else:
data_fields = data['fields'] = [fields[0]]
else:
data_fields = data['fields'] = [x for x in fields]
if request.order:
data['sort'] = [{x[0]: x[1]} for x in request.order]
if request.limit:
qfrom = None
qsize = None
if len(request.limit) > 1:
qfrom = request.limit.pop(0)
qsize = request.limit[0]
if qfrom is None:
data['size'] = qsize
elif qfrom >= 0:
data['from'] = qfrom
data['size'] = qsize
else:
#
# limit -1, 1000 => scan request, 1000 items at a time
#
params.update({
'search_type': 'scan',
'scroll': '10m',
'size': qsize
})
if calledFromREST:
return {'status': 'OK', 'result': '%s' % data}
else:
return request, data, params