def statlist():
# TODO add is_more here (as above)
""" Returns the counts and stats for the query """
auth, permitted = validate_user(mode="read")
try:
query = request.query_string
mode = parser.get_mode(query)
logging.debug('mode is %s' % mode)
default = {
"buckets": configM.searchfield(mode, 'statistics_buckets'),
"size": 100,
"cardinality": False
}
settings = parser.make_settings(permitted, default)
exclude = [] if auth else configM.searchfield(mode, 'secret_fields')
elasticq, more = parser.statistics(query,
settings,
exclude=exclude,
prefix='STAT_')
es = configM.elastic(mode=settings['mode'])
is_more = check_bucketsize(more, settings["size"], mode, es)
# TODO allow more than 100 000 hits here?
size = settings['size']
index, typ = configM.get_mode_index(settings['mode'])
ans = es.search(index=index,
body=loads(elasticq),
search_type="count",
size=size)
tables = []
for key, val in ans['aggregations']['q_statistics'].items():
if key.startswith('STAT_'):
tables.extend(generate_table(val, []))
# the length of tables might be longer than size, so truncate it
# generating shorter tables is not faster than generating all of it
# and then truncating
if size:
tables = tables[:size]
return jsonify({"stat_table": tables, "is_more": is_more})
except eh.KarpException as e: # pass on karp exceptions
logging.exception(e)
raise
except PErr.AuthenticationError as e:
logging.exception(e)
msg = e.message
raise eh.KarpAuthenticationError(msg)
except Exception as e: # catch *all* exceptions
# raise
logging.exception(e)
raise eh.KarpGeneralError("Unknown error", debug_msg=e, query=query)
def minientry():
""" Returns the counts and stats for the query """
max_page = configM.setupconfig['MINIENTRY_PAGE']
auth, permitted = validate_user(mode="read")
try:
query = request.query_string
mode = parser.get_mode(query)
default = {
'show': configM.searchfield(mode, 'minientry_fields'),
'size': 25
}
settings = parser.make_settings(permitted, default)
elasticq = parser.parse(query, settings)
show = settings['show']
if not auth:
# show = show - exclude
exclude = configM.searchfield(mode, 'secret_fields')
show = list(set(show).difference(exclude))
sort = sortorder(settings, mode, settings.get('query_command', ''))
start = settings['start'] if 'start' in settings else 0
es = configM.elastic(mode=settings['mode'])
index, typ = configM.get_mode_index(settings['mode'])
ans = parser.adapt_query(
settings['size'], start, es, loads(elasticq), {
'index': index,
'_source': show,
'from_': start,
'sort': sort,
'size': min(settings['size'], max_page),
'search_type': 'dfs_query_then_fetch'
})
if settings.get('highlight', False):
clean_highlight(ans)
return jsonify(ans)
except PErr.AuthenticationError as e:
logging.exception(e)
msg = e.message
raise eh.KarpAuthenticationError(msg)
except PErr.QueryError as e:
raise eh.KarpQueryError("Parse error, %s" % e.message,
debug_msg=e,
query=query)
except eh.KarpException as e: # pass on karp exceptions
logging.exception(e)
raise
except Exception as e: # catch *all* exceptions
logging.exception(e)
raise eh.KarpGeneralError("Unknown error", debug_msg=e, query=query)
def random():
auth, permitted = validate_user(mode="read")
try:
query = request.query_string
mode = parser.get_mode(query)
default = {
"show": configM.searchfield(mode, 'minientry_fields'),
"size": 1
}
settings = parser.make_settings(permitted, default)
elasticq = parser.random(query, settings)
es = configM.elastic(mode=mode)
index, typ = configM.get_mode_index(mode)
es_q = {
'index': index,
'body': loads(elasticq),
'size': settings['size']
}
if settings['show']:
show = settings['show']
if not auth:
# show = show - exclude
exclude = configM.searchfield(mode, 'secret_fields')
show = list(set(show).difference(exclude))
es_q['_source'] = show
ans = es.search(**es_q)
return jsonify(ans)
except PErr.AuthenticationError as e:
logging.exception(e)
msg = e.message
raise eh.KarpAuthenticationError(msg)
except eh.KarpException as e: # pass on karp exceptions
logging.exception(e)
raise
except Exception as e: # catch *all* exceptions
logging.exception(e)
raise eh.KarpGeneralError("Unknown error", debug_msg=e, query=query)
def formatpost():
""" Formats the posted data into wanted format
The data should be a list
Currently only working for saol
"""
# get and parse data
request.get_data()
data = request.data
try:
data = loads(data)
except ValueError as e:
raise eh.KarpParsingError(str(e))
# set all allowed lexicons (to avoid authentication exception
auth, permitted = validate_user(mode="read")
# find the wanted format
settings = parser.make_settings(permitted, {'size': 25})
query = request.query_string
parsed = parser.parse_qs(query)
parser.parse_extra(parsed, settings)
to_format = settings.get('format', '')
mode = parser.get_mode(query)
logging.debug('mode "%s"' % mode)
index, typ = configM.get_mode_index(mode)
if to_format:
if type(data) != list:
data = [data]
errmsg = 'Unkown format %s for mode %s' % (settings['format'], mode)
format_list = configM.extra_src(mode, 'format_list',
helpers.notdefined(errmsg))
ok, html = format_list(data, configM.elastic(mode=mode),
settings['format'], index)
return jsonify({'all': len(data), 'ok': ok, 'data': html})
else:
raise eh.KarpQueryError('Unkown format %s' % to_format)
def statistics():
""" Returns the counts and stats for the query """
auth, permitted = validate_user(mode="read")
try:
query = request.query_string
mode = parser.get_mode(query)
default = {
"buckets": configM.searchfield(mode, 'statistics_buckets'),
"size": 100,
"cardinality": False
}
settings = parser.make_settings(permitted, default)
exclude = [] if auth else configM.searchfield(mode, 'secret_fields')
elasticq, more = parser.statistics(query, settings, exclude=exclude)
es = configM.elastic(mode=settings['mode'])
is_more = check_bucketsize(more, settings, mode, es)
index, typ = configM.get_mode_index(settings['mode'])
# TODO allow more than 100 000 hits here?
ans = es.search(index=index,
body=loads(elasticq),
search_type="count",
size=settings['size'])
ans["is_more"] = is_more
return jsonify(ans)
except PErr.AuthenticationError as e:
logging.exception(e)
msg = e.message
raise eh.KarpAuthenticationError(msg)
except eh.KarpException as e: # pass on karp exceptions
logging.exception(e)
raise
except Exception as e: # catch *all* exceptions
logging.exception(e)
raise eh.KarpGeneralError("Unknown error", debug_msg=e, query=query)
def autocomplete():
""" Returns lemgrams matching the query text.
Each mode specifies in the configs which fields that should be
considered.
The parameter 'q' or 'query' is used when only one word form is to be
processed.
The parameter 'multi' is used when multiple word forms should be
processed.
The format of result depends on which flag that is set.
"""
auth, permitted = validate_user(mode="read")
query = request.query_string
try:
settings = parser.make_settings(permitted, {'size': 1000})
parsed = parser.parse_qs(query)
mode = parser.get_mode(query)
p_extra = parser.parse_extra(parsed, settings)
qs = parsed.get('q', []) or parsed.get('query', [])
multi = False
if not qs:
# check if there are multiple words forms to complete
qs = settings.get('multi', [])
logging.debug('qs %s' % qs)
multi = True
# use utf8, escape '"'
qs = [re.sub('"', '\\"', q.decode('utf8')) for q in qs]
headboost = configM.searchfield(mode, 'boosts')[0]
res = {}
ans = {}
# if multi is not true, only one iteration of this loop will be done
for q in qs:
boost = '''"functions": [{"boost_factor" : "500",
"filter":{"term":{"%s":"%s"}}}]''' % (headboost, q)
autocompleteq = configM.extra_src(mode, 'autocomplete',
autocompletequery)
exp = autocompleteq(mode, boost, q)
autocomplete_field = configM.searchonefield(
mode, 'autocomplete_field')
fields = ['"exists": {"field" : "%s"}' % autocomplete_field]
# last argument is the 'fields' used for highlightning
# TODO use filter?
elasticq = parser.search([exp] + p_extra,
fields,
'',
usefilter=True)
es = configM.elastic(mode=mode)
logging.debug('_source: %s' % autocomplete_field)
logging.debug(elasticq)
index, typ = configM.get_mode_index(mode)
ans = parser.adapt_query(
settings['size'], 0, es, loads(elasticq), {
'size': settings['size'],
'index': index,
'_source': autocomplete_field
})
# save the results for multi
res[q] = ans
if multi:
return jsonify(res)
else:
# single querys: only return the latest answer
return jsonify(ans)
except PErr.AuthenticationError as e:
logging.exception(e)
msg = e.message
raise eh.KarpAuthenticationError(msg)
except eh.KarpException as e: # pass on karp exceptions
logging.exception(e)
raise
except Exception as e: # catch *all* exceptions
logging.exception(e)
raise eh.KarpGeneralError("Unknown error", debug_msg=e, query=query)