def peek_query(args): if not isinstance(args, MultiDict): args = MultiDict(args) text = args.get('q', '').strip() q = text_query(text) filters = parse_filters(args) for entity in args.getlist('entity'): filters.append(('entities.id', entity)) q = filter_query(q, filters, []) q = add_filter(q, { 'not': { 'terms': { 'collection_id': authz.collections(authz.READ) } } }) q = { 'query': q, 'size': 0, 'aggregations': { 'collections': { 'terms': {'field': 'collection_id', 'size': 30} } }, '_source': False } # import json # print json.dumps(q, indent=2) result = get_es().search(index=get_es_index(), body=q, doc_type=TYPE_DOCUMENT) aggs = result.get('aggregations', {}).get('collections', {}) buckets = aggs.get('buckets', []) q = Collection.all_by_ids([b['key'] for b in buckets]) q = q.filter(Collection.creator_id != None) # noqa objs = {o.id: o for o in q.all()} roles = {} for bucket in buckets: collection = objs.get(bucket.get('key')) if collection is None or collection.private: continue if collection.creator_id in roles: roles[collection.creator_id]['total'] += bucket.get('doc_count') else: roles[collection.creator_id] = { 'name': collection.creator.name, 'email': collection.creator.email, 'total': bucket.get('doc_count') } roles = sorted(roles.values(), key=lambda r: r['total'], reverse=True) roles = [format_total(r) for r in roles] total = result.get('hits', {}).get('total') return format_total({ 'roles': roles, 'active': total > 0, 'total': total })
def alert_query(alert): """Execute the query and return a set of results.""" q = text_query(alert.query_text) q = authz_filter(q) if alert.entity_id: q = filter_query(q, [('entities.id', alert.entity_id)], OR_FIELDS) if alert.notified_at: q = add_filter(q, {"range": {"created_at": {"gt": alert.notified_at}}}) q = {'query': q, 'size': 150} result, hits, output = execute_basic(TYPE_DOCUMENT, q) collections = {} for doc in hits.get('hits', []): document = doc.get('_source') document['id'] = int(doc.get('_id')) document['collections'] = [] for coll in document['collection_id']: if coll not in authz.collections(authz.READ): continue if coll not in collections: collections[coll] = Collection.by_id(coll) if collections[coll] is None: continue document['collections'].append(collections[coll]) document['records'] = {'results': [], 'total': 0} output['results'].append(document) return output
def peek_query(args): if not isinstance(args, MultiDict): args = MultiDict(args) text = args.get('q', '').strip() q = text_query(text) filters = parse_filters(args) for entity in args.getlist('entity'): filters.append(('entities.id', entity)) q = filter_query(q, filters, []) q = add_filter( q, {'not': { 'terms': { 'collection_id': authz.collections(authz.READ) } }}) q = { 'query': q, 'size': 0, 'aggregations': { 'collections': { 'terms': { 'field': 'collection_id', 'size': 30 } } }, '_source': False } # import json # print json.dumps(q, indent=2) result = get_es().search(index=get_es_index(), body=q, doc_type=TYPE_DOCUMENT) aggs = result.get('aggregations', {}).get('collections', {}) buckets = aggs.get('buckets', []) q = Collection.all_by_ids([b['key'] for b in buckets]) q = q.filter(Collection.creator_id != None) # noqa objs = {o.id: o for o in q.all()} roles = {} for bucket in buckets: collection = objs.get(bucket.get('key')) if collection is None or collection.private: continue if collection.creator_id in roles: roles[collection.creator_id]['total'] += bucket.get('doc_count') else: roles[collection.creator_id] = { 'name': collection.creator.name, 'email': collection.creator.email, 'total': bucket.get('doc_count') } roles = sorted(roles.values(), key=lambda r: r['total'], reverse=True) roles = [format_total(r) for r in roles] total = result.get('hits', {}).get('total') return format_total({'roles': roles, 'active': total > 0, 'total': total})
def records_query(document_id, state): try: rows = [int(r) for r in state.getlist('row')] except: rows = [] score_query = state.has_text or len(rows) shoulds = records_query_shoulds(state) if not len(shoulds): shoulds = [match_all()] if len(rows): shoulds.append({ "constant_score": { "filter": { 'terms': { 'index': rows } }, "boost": 1000 } }) query = records_query_internal(document_id, shoulds, size=state.limit) query['query'] = filter_query(query['query'], state.filters) query['from'] = state.offset sort = [{'index': 'asc'}, {'page': 'asc'}] if score_query: sort.insert(0, '_score') return query
def alert_query(alert): """Execute the query and return a set of results.""" q = text_query(alert.query_text) q = authz_sources_filter(q) if alert.entity_id: q = filter_query(q, [('entities.uuid', alert.entity_id)], OR_FIELDS) if alert.notified_at: q = add_filter(q, {"range": {"created_at": {"gt": alert.notified_at}}}) q = {'query': q, 'size': 150} result, hits, output = execute_basic(TYPE_DOCUMENT, q) sub_queries = [] sources = {} for doc in hits.get('hits', []): document = doc.get('_source') document['id'] = int(doc.get('_id')) source_id = document['source_id'] if source_id not in sources: sources[source_id] = Source.by_id(source_id) if sources[source_id] is None: continue document['source'] = sources[source_id] document['records'] = {'results': [], 'total': 0} sq = records_query(document['id'], alert.to_query(), size=1) if sq is not None: sub_queries.append(json.dumps({})) sub_queries.append(json.dumps(sq)) output['results'].append(document) run_sub_queries(output, sub_queries) return output
def documents_iter(state, fields=None): """Iterate over a set of documents based on a query state.""" q = text_query(state.text) q = authz_filter(q, state.authz, roles=False) return scan_iter( { 'query': filter_query(q, state.filters), '_source': fields or DEFAULT_FIELDS }, TYPE_DOCUMENT)
def facet_collection(q, aggs, filters): aggs['scoped']['aggs']['collection'] = { 'filter': { 'query': filter_query(q, filters, OR_FIELDS, skip='collections') }, 'aggs': { 'collection': { 'terms': {'field': 'collections', 'size': 1000} } } } return aggs
def facet_source(q, aggs, filters): aggs['scoped']['aggs']['source'] = { 'filter': { 'query': filter_query(q, filters, OR_FIELDS, skip='source_id') }, 'aggs': { 'source': { 'terms': {'field': 'source_id', 'size': 1000} } } } return aggs
def facet_collections(q, aggs, filters): aggs['scoped']['aggs']['collections'] = { 'filter': { 'query': filter_query(q, filters, OR_FIELDS, skip='collection_id') }, 'aggs': { 'collections': { 'terms': {'field': 'collection_id', 'size': 1000} } } } return aggs
def peek_query(state): """Peek into hidden collections. This allows users to retrieve an approximate result count of a given query against those collections which they are not authorised to view. It is a rudimentary collaboration mechanism. """ filters = state.filters cq = Collection.all() cq = cq.filter(not_(Collection.id.in_(state.authz.collections_read))) cq = cq.filter(Collection.creator_id != None) # noqa cq = cq.filter(Collection.private != True) # noqa collections = {c.id: c for c in cq} filters['collection_id'] = collections.keys() q = text_query(state.text) q = { 'query': filter_query(q, filters), 'query': q, 'size': 0, 'aggregations': { 'collections': { 'terms': {'field': 'collection_id', 'size': 1000} } }, '_source': False } result = es.search(index=es_index, body=q, doc_type=TYPE_DOCUMENT) roles = {} total = 0 aggs = result.get('aggregations', {}).get('collections', {}) for bucket in aggs.get('buckets', []): collection = collections.get(bucket.get('key')) if collection is None or collection.creator is None: continue total += bucket.get('doc_count') if collection.creator_id in roles: roles[collection.creator_id]['total'] += bucket.get('doc_count') else: roles[collection.creator_id] = { 'name': collection.creator.name, 'email': collection.creator.email, 'total': bucket.get('doc_count') } roles = sorted(roles.values(), key=lambda r: r['total'], reverse=True) roles = [format_total(r) for r in roles] return format_total({ 'roles': roles, 'active': total > 0, 'total': total })
def facet_collections(state, q, aggs): filters = state.filters filters['collection_id'] = state.authz.collections_read aggs['scoped']['aggs']['collections'] = { 'filter': { 'query': filter_query(q, filters) }, 'aggs': { 'collections': { 'terms': {'field': 'collection_id', 'size': state.facet_size} } } } return aggs
def entities_query(args, fields=None, facets=True): """Parse a user query string, compose and execute a query.""" if not isinstance(args, MultiDict): args = MultiDict(args) text = args.get('q', '').strip() if text is None or not len(text): q = match_all() else: q = { "query_string": { "query": text, "fields": [ 'name^15', 'name_latin^5', 'terms^12', 'terms_latin^3', 'summary^10', 'summary_latin^7', 'description^5', 'description_latin^3' ], "default_operator": "AND", "use_dis_max": True } } q = authz_filter(q) filters = parse_filters(args) aggs = {'scoped': {'global': {}, 'aggs': {}}} if facets: facets = args.getlist('facet') if 'collections' in facets: aggs = facet_collections(q, aggs, filters) facets.remove('collections') aggs = aggregate(q, aggs, facets) sort_mode = args.get('sort', '').strip().lower() default_sort = 'score' if len(text) else 'doc_count' sort_mode = sort_mode or default_sort if sort_mode == 'doc_count': sort = [{'doc_count': 'desc'}, '_score'] elif sort_mode == 'alphabet': sort = [{'name': 'asc'}, '_score'] elif sort_mode == 'score': sort = ['_score'] return { 'sort': sort, 'query': filter_query(q, filters, OR_FIELDS), 'aggregations': aggs, '_source': fields or DEFAULT_FIELDS }
def facet_source(q, aggs, filters): aggs['scoped']['aggs']['source'] = { 'filter': { 'query': filter_query(q, filters, OR_FIELDS, skip='source_id') }, 'aggs': { 'source': { 'terms': { 'field': 'source_id', 'size': 1000 } } } } return aggs
def documents_query(args, fields=None, facets=True, escape=True): """Parse a user query string, compose and execute a query. Escape -- filter input so it can be safely used in an ES query string """ if not isinstance(args, MultiDict): args = MultiDict(args) text = args.get('q', '').strip() if escape: text = escape_query_string(text) q = text_query(text) q = authz_sources_filter(q) # Sorting -- should this be passed into search directly, instead of # these aliases? sort_mode = args.get('sort', '').strip().lower() if sort_mode == 'score': sort = ['_score'] elif sort_mode == 'newest_filed': sort = [{'filing_date': 'desc'}] elif sort_mode == 'oldest_filed': sort = [{'filing_date': 'asc'}] elif sort_mode == 'newest_added': sort = [{'dates': 'desc'}, {'created_at': 'desc'}, '_score'] elif sort_mode == 'oldest_added': sort = [{'dates': 'asc'}, {'created_at': 'asc'}, '_score'] else: sort = [{'updated_at': 'desc'}, {'created_at': 'desc'}, '_score'] filters = parse_filters(args) for entity in args.getlist('entity'): filters.append(('entities.uuid', entity)) aggs = {} if facets: aggs = aggregate(q, args) aggs = facet_source(q, aggs, filters) #aggs = facet_sector(q, aggs, filters) q = entity_collections(q, aggs, args, filters) # XXX this is where I should be hooking in openoil aggregations signals.document_query_process.send(q=q, args=args) return { 'sort': sort, 'query': filter_query(q, filters, OR_FIELDS), 'aggregations': aggs, '_source': fields or DEFAULT_FIELDS #'fields': fields or DEFAULT_FIELDS }
def links_query(origin, state): """Parse a user query string, compose and execute a query.""" if state.has_text: q = { "query_string": { "query": state.text, "fields": ['name^5', 'names^2', 'text'], "default_operator": "AND", "use_dis_max": True } } else: q = match_all() ids = origin.get('ids') or [origin.get('id')] q = add_filter(q, {'terms': {'origin.id': ids}}) q = authz_filter(q, state.authz, roles=True) aggs = {'scoped': {'global': {}, 'aggs': {}}} aggs = aggregate(state, q, aggs, state.facet_names) if state.sort == 'score': sort = ['_score'] else: sort = [{ 'properties.start_date': 'desc' }, { 'properties.end_date': 'desc' }] q = { 'sort': sort, 'query': filter_query(q, state.filters), 'aggregations': aggs, 'size': state.limit, 'from': state.offset, '_source': DEFAULT_FIELDS } result, hits, output = execute_basic(TYPE_LINK, q) output['facets'] = parse_facet_result(state, result) for doc in hits.get('hits', []): link = doc.get('_source') link['id'] = doc.get('_id') link['score'] = doc.get('_score') output['results'].append(link) return output
def entities_query(args, fields=None, facets=True): """Parse a user query string, compose and execute a query.""" if not isinstance(args, MultiDict): args = MultiDict(args) text = args.get('q', '').strip() if text is None or not len(text): q = match_all() else: q = { "query_string": { "query": text, "fields": ['name^15', 'name_latin^5', 'terms^12', 'terms_latin^3', 'summary^10', 'summary_latin^7', 'description^5', 'description_latin^3'], "default_operator": "AND", "use_dis_max": True } } q = authz_filter(q) filters = parse_filters(args) aggs = {'scoped': {'global': {}, 'aggs': {}}} if facets: facets = args.getlist('facet') if 'collections' in facets: aggs = facet_collections(q, aggs, filters) facets.remove('collections') aggs = aggregate(q, aggs, facets) sort_mode = args.get('sort', '').strip().lower() default_sort = 'score' if len(text) else 'doc_count' sort_mode = sort_mode or default_sort if sort_mode == 'doc_count': sort = [{'doc_count': 'desc'}, '_score'] elif sort_mode == 'alphabet': sort = [{'name': 'asc'}, '_score'] elif sort_mode == 'score': sort = ['_score'] return { 'sort': sort, 'query': filter_query(q, filters, OR_FIELDS), 'aggregations': aggs, '_source': fields or DEFAULT_FIELDS }
def leads_query(collection_id, state): q = {'term': {'entity_collection_id': collection_id}} q = authz_filter(q, state.authz, roles=True) aggs = {'scoped': {'global': {}, 'aggs': {}}} facets = list(state.facet_names) if 'collections' in facets: aggs = facet_collections(state, q, aggs) facets.remove('collections') aggs = aggregate(state, q, aggs, facets) q = { 'sort': [{ 'judgement': 'asc' }, { 'score': 'desc' }, { 'match_id': 'asc' }], 'query': filter_query(q, state.filters), 'aggregations': aggs, 'size': state.limit, 'from': state.offset } result, hits, output = execute_basic(TYPE_LEAD, q) output['facets'] = parse_facet_result(state, result) entities = set([]) for doc in hits.get('hits', []): link = doc.get('_source') link['id'] = doc.get('_id') entities.add(link.get('entity_id')) entities.add(link.get('match_id')) output['results'].append(link) q = {'terms': {'_id': list(entities)}} q = {'query': q, 'size': len(entities) + 2} _, hits, _ = execute_basic(TYPE_ENTITY, q) for doc in hits.get('hits', []): entity = doc.get('_source') entity['id'] = doc.get('_id') for result in output['results']: if result.get('match_id') == entity['id']: result['match'] = entity if result.get('entity_id') == entity['id']: result['entity'] = entity return output
def alert_query(alert): """Execute the query and return a set of results.""" q = text_query(alert.query_text) q = authz_filter(q) if alert.entity_id: q = filter_query(q, [('entities.id', alert.entity_id)], OR_FIELDS) if alert.notified_at: q = add_filter(q, { "range": { "created_at": { "gt": alert.notified_at } } }) q = { 'query': q, 'size': 150 } result, hits, output = execute_basic(TYPE_DOCUMENT, q) sub_queries = [] collections = {} for doc in hits.get('hits', []): document = doc.get('_source') document['id'] = int(doc.get('_id')) document['collections'] = [] for coll in document['collection_id']: if coll not in authz.collections(authz.READ): continue if coll not in collections: collections[coll] = Collection.by_id(coll) if collections[coll] is None: continue document['collections'].append(collections[coll]) document['records'] = {'results': [], 'total': 0} sq = records_query(document['id'], alert.to_query(), size=1) if sq is not None: sub_queries.append(json.dumps({})) sub_queries.append(json.dumps(sq)) output['results'].append(document) run_sub_queries(output, sub_queries) return output
def documents_query(args, fields=None, facets=True): """Parse a user query string, compose and execute a query.""" if not isinstance(args, MultiDict): args = MultiDict(args) text = args.get('q', '').strip() q = text_query(text) q = authz_filter(q) # Sorting -- should this be passed into search directly, instead of # these aliases? sort_mode = args.get('sort', '').strip().lower() if text or sort_mode == 'score': sort = ['_score'] elif sort_mode == 'newest': sort = [{'dates': 'desc'}, {'created_at': 'desc'}, '_score'] elif sort_mode == 'oldest': sort = [{'dates': 'asc'}, {'created_at': 'asc'}, '_score'] else: sort = [{'updated_at': 'desc'}, {'created_at': 'desc'}, '_score'] filters = parse_filters(args) for entity in args.getlist('entity'): filters.append(('entities.id', entity)) aggs = {'scoped': {'global': {}, 'aggs': {}}} if facets: facets = args.getlist('facet') if 'collections' in facets: aggs = facet_collections(q, aggs, filters) facets.remove('collections') if 'entities' in facets: aggs = facet_entities(aggs, args) facets.remove('entities') aggs = aggregate(q, aggs, facets) signals.document_query_process.send(q=q, args=args) return { 'sort': sort, 'query': filter_query(q, filters, OR_FIELDS), 'aggregations': aggs, '_source': fields or DEFAULT_FIELDS }
def alert_query(alert): """Execute the query and return a set of results.""" q = text_query(alert.query_text) q = authz_sources_filter(q) if alert.entity_id: q = filter_query(q, [('entities.uuid', alert.entity_id)], OR_FIELDS) if alert.notified_at: q = add_filter(q, { "range": { "created_at": { "gt": alert.notified_at } } }) q = { 'query': q, 'size': 150 } result, hits, output = execute_basic(TYPE_DOCUMENT, q) sub_queries = [] sources = {} for doc in hits.get('hits', []): document = doc.get('_source') document['id'] = int(doc.get('_id')) source_id = document['source_id'] if source_id not in sources: sources[source_id] = Source.by_id(source_id) if sources[source_id] is None: continue document['source'] = sources[source_id] document['records'] = {'results': [], 'total': 0} sq = records_query(document['id'], alert.to_query(), size=1) if sq is not None: sub_queries.append(json.dumps({})) sub_queries.append(json.dumps(sq)) output['results'].append(document) run_sub_queries(output, sub_queries) return output
def entities_query(state, fields=None, facets=True, doc_counts=False): """Parse a user query string, compose and execute a query.""" if state.has_text: q = { "query_string": { "query": state.text, "fields": ['name^5', 'names^2', 'text'], "default_operator": "AND", "use_dis_max": True } } else: q = match_all() if state.raw_query: q = {"bool": {"must": [q, state.raw_query]}} q = authz_filter(q, state.authz, roles=True) aggs = {'scoped': {'global': {}, 'aggs': {}}} if facets: facets = list(state.facet_names) if 'collections' in facets: aggs = facet_collections(state, q, aggs) facets.remove('collections') aggs = aggregate(state, q, aggs, facets) if state.sort == 'doc_count': sort = [{'doc_count': 'desc'}, '_score'] elif state.sort == 'score': sort = ['_score', {'name_sort': 'asc'}] else: sort = [{'name_sort': 'asc'}] # pprint(q) q = { 'sort': sort, 'query': filter_query(q, state.filters), 'aggregations': aggs, 'size': state.limit, 'from': state.offset, '_source': fields or DEFAULT_FIELDS } result, hits, output = execute_basic(TYPE_ENTITY, q) output['facets'] = parse_facet_result(state, result) sub_queries = [] for doc in hits.get('hits', []): entity = doc.get('_source') entity['id'] = doc.get('_id') entity['score'] = doc.get('_score') entity['api_url'] = url_for('entities_api.view', id=doc.get('_id')) output['results'].append(entity) sq = {'term': {'entities.id': entity['id']}} sq = add_filter( sq, {'terms': { 'collection_id': state.authz.collections_read }}) sq = {'size': 0, 'query': sq} sub_queries.append(json.dumps({})) sub_queries.append(json.dumps(sq)) if doc_counts and len(sub_queries): # Get the number of matching documents for each entity. body = '\n'.join(sub_queries) res = es.msearch(index=es_index, doc_type=TYPE_DOCUMENT, body=body) for (entity, res) in zip(output['results'], res.get('responses')): entity['doc_count'] = res.get('hits', {}).get('total') return output
def documents_query(state, fields=None, facets=True, since=None): """Parse a user query string, compose and execute a query.""" # This used to be several functions, but it's actually incredibly # procedural and so it's been linearised into one function. To really # clean this up, I think it should be based around an object model of # some sort. q = text_query(state.text) if state.raw_query: q = {"bool": {"must": [q, state.raw_query]}} q = authz_filter(q, state.authz, roles=False) # Used by alerting to find only updated results: if since is not None: q = add_filter(q, {"range": {"created_at": {"gt": since}}}) # Sorting if state.sort == 'newest': sort = [{'dates': 'desc'}, {'created_at': 'desc'}, '_score'] if state.sort == 'oldest': sort = [{'dates': 'asc'}, {'created_at': 'asc'}, '_score'] else: sort = ['_score'] # TODO: find a better way to handle "slightly special" aggregations like # entities and collections. aggs = {'scoped': {'global': {}, 'aggs': {}}} if facets: facets = list(state.facet_names) if 'collections' in facets: aggs = facet_collections(state, q, aggs) facets.remove('collections') if 'entities' in facets: aggs = facet_entities(state, aggs) facets.remove('entities') aggs = aggregate(state, q, aggs, facets) # allow plug-ins to post-process the query. signals.document_query_process.send(q=q, state=state) q = { 'sort': sort, 'size': state.limit, 'from': state.offset, 'query': filter_query(q, state.filters), 'aggregations': aggs, '_source': fields or DEFAULT_FIELDS } result, hits, output = execute_basic(TYPE_DOCUMENT, q) # This will add labels and other contextual information. output['facets'] = parse_facet_result(state, result) # After the main query has run, a sub-query will be run for each returned # result in order to find relevant records for result highlighting. sub_shoulds = records_query_shoulds(state) sub_queries = [] for doc in hits.get('hits', []): document = doc.get('_source') document['id'] = int(doc.get('_id')) document['score'] = doc.get('_score') document['records'] = {'results': [], 'total': 0} collection_id = document.get('collection_id') try: # FIXME: there's got to be a nicer way of doing this.... document['public'] = state.authz.collection_public(collection_id) except: document['public'] = None sq = records_query_internal(document['id'], sub_shoulds) if sq is not None: sub_queries.append(json.dumps({})) sub_queries.append(json.dumps(sq)) output['results'].append(document) if not len(sub_queries): return output body = '\n'.join(sub_queries) res = es.msearch(index=es_index, doc_type=TYPE_RECORD, body=body) for doc in output['results']: for sq in res.get('responses', []): sqhits = sq.get('hits', {}) doc['records']['total'] = sqhits.get('total', 0) for hit in sqhits.get('hits', {}): record = hit.get('_source') if doc['id'] != record['document_id']: continue hlt = hit.get('highlight', {}) texts = hlt.get('text', []) or hlt.get('text_latin', []) texts = [clean_highlight(t) for t in texts] texts = [t for t in texts if len(t)] if len(texts): record['text'] = texts[0] doc['records']['results'].append(record) return output