def documents_iter(state, fields=None):
"""Iterate over a set of documents based on a query state."""
q = text_query(state.text)
q = authz_filter(q, state.authz, roles=False)
return scan_iter(
{
'query': filter_query(q, state.filters),
'_source': fields or DEFAULT_FIELDS
}, TYPE_DOCUMENT)
def peek_query(state):
"""Peek into hidden collections.
This allows users to retrieve an approximate result count of a given query
against those collections which they are not authorised to view. It is a
rudimentary collaboration mechanism.
"""
filters = state.filters
cq = Collection.all()
cq = cq.filter(not_(Collection.id.in_(state.authz.collections_read)))
cq = cq.filter(Collection.creator_id != None) # noqa
cq = cq.filter(Collection.private != True) # noqa
collections = {c.id: c for c in cq}
filters['collection_id'] = collections.keys()
q = text_query(state.text)
q = {
'query': filter_query(q, filters),
'query': q,
'size': 0,
'aggregations': {
'collections': {
'terms': {'field': 'collection_id', 'size': 1000}
}
},
'_source': False
}
result = es.search(index=es_index, body=q, doc_type=TYPE_DOCUMENT)
roles = {}
total = 0
aggs = result.get('aggregations', {}).get('collections', {})
for bucket in aggs.get('buckets', []):
collection = collections.get(bucket.get('key'))
if collection is None or collection.creator is None:
continue
total += bucket.get('doc_count')
if collection.creator_id in roles:
roles[collection.creator_id]['total'] += bucket.get('doc_count')
else:
roles[collection.creator_id] = {
'name': collection.creator.name,
'email': collection.creator.email,
'total': bucket.get('doc_count')
}
roles = sorted(roles.values(), key=lambda r: r['total'], reverse=True)
roles = [format_total(r) for r in roles]
return format_total({
'roles': roles,
'active': total > 0,
'total': total
})
def documents_query(state, fields=None, facets=True, since=None):
"""Parse a user query string, compose and execute a query."""
# This used to be several functions, but it's actually incredibly
# procedural and so it's been linearised into one function. To really
# clean this up, I think it should be based around an object model of
# some sort.
q = text_query(state.text)
if state.raw_query:
q = {"bool": {"must": [q, state.raw_query]}}
q = authz_filter(q, state.authz, roles=False)
# Used by alerting to find only updated results:
if since is not None:
q = add_filter(q, {"range": {"created_at": {"gt": since}}})
# Sorting
if state.sort == 'newest':
sort = [{'dates': 'desc'}, {'created_at': 'desc'}, '_score']
if state.sort == 'oldest':
sort = [{'dates': 'asc'}, {'created_at': 'asc'}, '_score']
else:
sort = ['_score']
# TODO: find a better way to handle "slightly special" aggregations like
# entities and collections.
aggs = {'scoped': {'global': {}, 'aggs': {}}}
if facets:
facets = list(state.facet_names)
if 'collections' in facets:
aggs = facet_collections(state, q, aggs)
facets.remove('collections')
if 'entities' in facets:
aggs = facet_entities(state, aggs)
facets.remove('entities')
aggs = aggregate(state, q, aggs, facets)
# allow plug-ins to post-process the query.
signals.document_query_process.send(q=q, state=state)
q = {
'sort': sort,
'size': state.limit,
'from': state.offset,
'query': filter_query(q, state.filters),
'aggregations': aggs,
'_source': fields or DEFAULT_FIELDS
}
result, hits, output = execute_basic(TYPE_DOCUMENT, q)
# This will add labels and other contextual information.
output['facets'] = parse_facet_result(state, result)
# After the main query has run, a sub-query will be run for each returned
# result in order to find relevant records for result highlighting.
sub_shoulds = records_query_shoulds(state)
sub_queries = []
for doc in hits.get('hits', []):
document = doc.get('_source')
document['id'] = int(doc.get('_id'))
document['score'] = doc.get('_score')
document['records'] = {'results': [], 'total': 0}
collection_id = document.get('collection_id')
try:
# FIXME: there's got to be a nicer way of doing this....
document['public'] = state.authz.collection_public(collection_id)
except:
document['public'] = None
sq = records_query_internal(document['id'], sub_shoulds)
if sq is not None:
sub_queries.append(json.dumps({}))
sub_queries.append(json.dumps(sq))
output['results'].append(document)
if not len(sub_queries):
return output
body = '\n'.join(sub_queries)
res = es.msearch(index=es_index, doc_type=TYPE_RECORD, body=body)
for doc in output['results']:
for sq in res.get('responses', []):
sqhits = sq.get('hits', {})
doc['records']['total'] = sqhits.get('total', 0)
for hit in sqhits.get('hits', {}):
record = hit.get('_source')
if doc['id'] != record['document_id']:
continue
hlt = hit.get('highlight', {})
texts = hlt.get('text', []) or hlt.get('text_latin', [])
texts = [clean_highlight(t) for t in texts]
texts = [t for t in texts if len(t)]
if len(texts):
record['text'] = texts[0]
doc['records']['results'].append(record)
return output