def append_search_filters(self, value, node, query, request): try: if value["val"] != "": match_query = Match(field="tiles.data.%s" % (str(node.pk)), type="phrase", query=value["val"]) if "!" in value["op"]: query.must_not(match_query) query.filter(Exists(field="tiles.data.%s" % (str(node.pk)))) else: query.must(match_query) except KeyError as e: pass
def append_search_filters(self, value, node, query, request): try: if value['val'] != '': match_query = Match(field='tiles.data.%s' % (str(node.pk)), type="phrase", query=value['val']) if '!' in value['op']: query.must_not(match_query) query.filter(Exists(field="tiles.data.%s" % (str(node.pk)))) else: query.must(match_query) except KeyError, e: pass
def get_indexed_concepts(se, conceptid, concept_value): """ Searches for a conceptid from the database and confirms that the database concept value matches the indexed value """ result = 'failed: cannot find' + conceptid query = Query(se, start=0, limit=100) phrase = Match(field='conceptid', query=conceptid, type='phrase_prefix') query.add_query(phrase) results = query.search(index='concept_labels') if len(results['hits']['hits']) > 0: source = results['hits']['hits'][0]['_source'] if conceptid == source['conceptid'] or concept_value == source['value']: result = 'passed' else: result = 'failed: concept value does not match' return result
def append_search_filters(self, value, node, query, request): # Match the label in the same manner as a String datatype try: if value["val"] != "": match_type = "phrase_prefix" if "~" in value["op"] else "phrase" if "~" in value["op"]: match_query = Match( field="tiles.data.%s.url" % (str(node.pk)), query=value["val"], type=match_type, ) if "=" in value["op"]: match_query = Term(field="tiles.data.%s.url.keyword" % (str(node.pk)), term=value["val"]) if "!" in value["op"]: query.must_not(match_query) query.filter(Exists(field="tiles.data.%s" % (str(node.pk)))) else: query.must(match_query) except KeyError as e: pass
def append_search_filters(self, value, node, query, request): try: if value["op"] == "null" or value["op"] == "not_null": self.append_null_search_filters(value, node, query, request) elif value["val"] != "": base_query = Bool() base_query.filter( Terms(field="graph_id", terms=[str(node.graph_id)])) match_query = Nested(path="tiles", query=Match(field="tiles.data.%s" % (str(node.pk)), type="phrase", query=value["val"])) if "!" in value["op"]: base_query.must_not(match_query) # base_query.filter(Exists(field="tiles.data.%s" % (str(node.pk)))) else: base_query.must(match_query) query.must(base_query) except KeyError as e: pass
def test_bulk_delete(self): """ Test bulk deleting of documents in Elasticsearch """ se = SearchEngineFactory().create() # se.create_index(index='test') for i in range(10): x = { 'id': i, 'type': 'prefLabel', 'value': 'test pref label', } se.index_data(index='test', doc_type='test', body=x, idfield='id', refresh=True) y = { 'id': i + 100, 'type': 'altLabel', 'value': 'test alt label', } se.index_data(index='test', doc_type='test', body=y, idfield='id', refresh=True) query = Query(se, start=0, limit=100) match = Match(field='type', query='altLabel') query.add_query(match) query.delete(index='test', refresh=True) self.assertEqual( se.es.count(index='test', doc_type='test')['count'], 10)
def test_delete_by_query(self): """ Test deleting documents by query in Elasticsearch """ se = SearchEngineFactory().create() for i in range(10): x = {"id": i, "type": "prefLabel", "value": "test pref label"} se.index_data(index="test", body=x, idfield="id", refresh=True) y = {"id": i + 100, "type": "altLabel", "value": "test alt label"} se.index_data(index="test", body=y, idfield="id", refresh=True) time.sleep(1) query = Query(se, start=0, limit=100) match = Match(field="type", query="altLabel") query.add_query(match) query.delete(index="test", refresh=True) self.assertEqual(se.count(index="test"), 10)
def build_search_results_dsl(request): # Results are sorted ascendingly by the value of SITE_ID.E42, which is displayed as primary name of Heritage Resources. # Must go back to this method once new Automatic Resource ID has been fully developed (AZ 10/08/16) Update 06/09/16: EAMENA_ID.E42 now used as sorting criterion. sorting = { "child_entities.label": { "order" : "asc", "nested_path": "child_entities", "nested_filter": { "term": {"child_entities.entitytypeid" : "EAMENA_ID.E42"} } } } term_filter = request.GET.get('termFilter', '') spatial_filter = JSONDeserializer().deserialize(request.GET.get('spatialFilter', None)) export = request.GET.get('export', None) page = 1 if request.GET.get('page') == '' else int(request.GET.get('page', 1)) temporal_filter = JSONDeserializer().deserialize(request.GET.get('temporalFilter', None)) boolean_search = request.GET.get('booleanSearch', '') filter_and_or = JSONDeserializer().deserialize(request.GET.get('termFilterAndOr', '')) filter_grouping = JSONDeserializer().deserialize(request.GET.get('termFilterGroup', '')) filter_combine_flags = JSONDeserializer().deserialize(request.GET.get('termFilterCombineWithPrev', '')) #Ignore first entry as it is a dummy filter_combine_flags = filter_combine_flags[1:] # filter_combine_flags = [False, True, False, False, False] # filter_groups = JSONDeserializer().deserialize(request.GET.get('termFilterGroups', '')) # Not here yet, so put in some bogus data # filter_groups = [ # 'NAME.E41', # 'NAME.E41', # 'DISTURBANCE_STATE.E3', # 'THREAT_STATE.E3' # ] se = SearchEngineFactory().create() if export != None: limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE else: limit = settings.SEARCH_ITEMS_PER_PAGE query = Query(se, start=limit*int(page-1), limit=limit) boolquery = Bool() boolfilter = Bool() is_empty_temporal_filter = True # store each search term in an initially. These will be combined based on the global and/or and the optional groupings terms_queries = []; # logging.warning("-------QUERY-------") if term_filter != '' or not is_empty_temporal_filter: for index, select_box in enumerate(JSONDeserializer().deserialize(term_filter)): selectbox_boolfilter = Bool() groupid = filter_grouping[index] if not groupid == 'No group': # build a nested query against the nested_entities # build a nested query for each resource type for resourcetype in settings.RESOURCE_TYPE_CONFIGS().keys(): # trace the path from each term to the group root term_paths = [] for term in select_box: # trace path from group root to this term if term['type'] == 'concept': # get all the parent concepts for this value i.e. the field concept_relations = models.ConceptRelations.objects.filter(conceptidto=term['value'], relationtype="member") for relation in concept_relations: term_parent_concept = models.Concepts.objects.get(conceptid=relation.conceptidfrom) # get the steps from the root to that concept if term_parent_concept.nodetype.nodetype == "Collection": term_schema = Entity.get_mapping_schema_to(term_parent_concept.legacyoid) elif term_parent_concept.nodetype.nodetype == 'Concept': # need to get at the parent until we reach the root collection. concepts are arranged hierarchically parent_relations_to = models.ConceptRelations.objects.filter(conceptidto=term_parent_concept.conceptid, relationtype='member') grandparent = models.Concepts.objects.filter(conceptid=parent_relations_to[0].conceptidfrom) term_schema = Entity.get_mapping_schema_to(grandparent[0].legacyoid) #this path begins at the root, and ends up at the node in question if resourcetype in term_schema: term_path = term_schema[resourcetype]['steps'] term_paths.append({ 'term': term, 'path': term_path }) break elif term['type'] == 'term': concept = models.Concepts.objects.get(conceptid=term['context']) term_schema = Entity.get_mapping_schema_to(concept.legacyoid) if resourcetype in term_schema: term_path = term_schema[resourcetype]['steps'] term_paths.append({ 'term': term, 'path': term_path }) elif term['type'] == 'string': term_schema = Entity.get_mapping_schema_to(groupid) if resourcetype in term_schema: term_path = term_schema[resourcetype]['steps'] term_paths.append({ 'term': term, 'path': term_path }) if 'year_min_max' in temporal_filter[index] and len(temporal_filter[index]['year_min_max']) == 2: start_date = date(temporal_filter[index]['year_min_max'][0], 1, 1) end_date = date(temporal_filter[index]['year_min_max'][1], 12, 31) if start_date: start_date = start_date.isoformat() if end_date: end_date = end_date.isoformat() if 'inverted' not in temporal_filter[index]: inverted_temporal_filter = False else: if temporal_filter[index]['inverted']: inverted_temporal_filter = True else: inverted_temporal_filter = False term_paths.append({ 'term': { 'date_operator': '3', 'start_date': start_date, 'end_date': end_date, 'type': 'date', 'inverted': inverted_temporal_filter }, 'path': term_path }) if 'filters' in temporal_filter[index]: term_schema = Entity.get_mapping_schema_to(groupid) if resourcetype in term_schema: term_path = term_schema[resourcetype]['steps'] for temporal_filter_item in temporal_filter[index]['filters']: date_type = '' searchdate = '' date_operator = '' for node in temporal_filter_item['nodes']: if node['entitytypeid'] == 'DATE_COMPARISON_OPERATOR.E55': date_operator = node['value'] elif node['entitytypeid'] == 'date': searchdate = node['value'] else: date_type = node['value'] date_value = datetime.strptime(searchdate, '%Y-%m-%d').isoformat() if 'inverted' not in temporal_filter[index]: inverted_temporal_filter = False else: if temporal_filter[index]['inverted']: inverted_temporal_filter = True else: inverted_temporal_filter = False term_paths.append({ 'term': { 'date_operator': date_operator, 'date_value': date_value, 'type': 'date', 'inverted': inverted_temporal_filter }, 'path': term_path }) # combine the traced path to build a nested query group_query = nested_query_from_pathed_values(term_paths, 'nested_entity.child_entities') # add nested query to overall query selectbox_boolfilter.should(group_query) # logging.warning("BOX QUERY - %s", JSONSerializer().serialize(selectbox_boolfilter, indent=2)) else: for term in select_box: if term['type'] == 'term': entitytype = models.EntityTypes.objects.get(conceptid_id=term['context']) boolfilter_nested = Bool() boolfilter_nested.must(Terms(field='child_entities.entitytypeid', terms=[entitytype.pk])) boolfilter_nested.must(Match(field='child_entities.value', query=term['value'], type='phrase')) nested = Nested(path='child_entities', query=boolfilter_nested) if filter_and_or[index] == 'or': if not term['inverted']: selectbox_boolfilter.should(nested) else: if term['inverted']: selectbox_boolfilter.must_not(nested) else: selectbox_boolfilter.must(nested) elif term['type'] == 'concept': concept_ids = _get_child_concepts(term['value']) terms = Terms(field='domains.conceptid', terms=concept_ids) nested = Nested(path='domains', query=terms) if filter_and_or[index] == 'or': if not term['inverted']: selectbox_boolfilter.should(nested) else: if term['inverted']: selectbox_boolfilter.must_not(nested) else: selectbox_boolfilter.must(nested) elif term['type'] == 'string': boolquery2 = Bool() #This bool contains the subset of nested string queries on both domains and child_entities paths boolfilter_folded = Bool() #This bool searches by string in child_entities, where free text strings get indexed boolfilter_folded2 = Bool() #This bool searches by string in the domains path,where controlled vocabulary concepts get indexed boolfilter_folded.should(Match(field='child_entities.value', query=term['value'], type='phrase_prefix', fuzziness='AUTO', operator='and')) boolfilter_folded.should(Match(field='child_entities.value.folded', query=term['value'], type='phrase_prefix', fuzziness='AUTO', operator='and')) boolfilter_folded.should(Match(field='child_entities.value.folded', query=term['value'], fuzziness='AUTO', operator='and')) nested = Nested(path='child_entities', query=boolfilter_folded) boolfilter_folded2.should(Match(field='domains.label', query=term['value'], type='phrase_prefix', fuzziness='AUTO', operator='and')) boolfilter_folded2.should(Match(field='domains.label.folded', query=term['value'], type='phrase_prefix', fuzziness='AUTO', operator='and')) boolfilter_folded2.should(Match(field='domains.label.folded', query=term['value'], fuzziness='AUTO', operator='and')) nested2 = Nested(path='domains', query=boolfilter_folded2) boolquery2.should(nested) boolquery2.should(nested2) if filter_and_or[index] == 'or': if not term['inverted']: # use boolfilter here instead of boolquery because boolquery # can't be combined with other boolfilters using boolean OR selectbox_boolfilter.should(boolquery2) else: if term['inverted']: selectbox_boolfilter.must_not(boolquery2) else: selectbox_boolfilter.must(boolquery2) if 'year_min_max' in temporal_filter[index] and len(temporal_filter[index]['year_min_max']) == 2: start_date = date(temporal_filter[index]['year_min_max'][0], 1, 1) end_date = date(temporal_filter[index]['year_min_max'][1], 12, 31) if start_date: start_date = start_date.isoformat() if end_date: end_date = end_date.isoformat() range = Range(field='dates.value', gte=start_date, lte=end_date) nested = Nested(path='dates', query=range) if 'inverted' not in temporal_filter[index]: temporal_filter[index]['inverted'] = False if temporal_filter[index]['inverted']: selectbox_boolfilter.must_not(nested) else: selectbox_boolfilter.must(nested) if 'filters' in temporal_filter[index]: for temporal_filter_item in temporal_filter[index]['filters']: date_type = '' searchdate = '' date_operator = '' for node in temporal_filter_item['nodes']: if node['entitytypeid'] == 'DATE_COMPARISON_OPERATOR.E55': date_operator = node['value'] elif node['entitytypeid'] == 'date': searchdate = node['value'] else: date_type = node['value'] date_value = datetime.strptime(searchdate, '%Y-%m-%d').isoformat() if date_operator == '1': # equals query range = Range(field='dates.value', gte=date_value, lte=date_value) elif date_operator == '0': # greater than query range = Range(field='dates.value', lt=date_value) elif date_operator == '2': # less than query range = Range(field='dates.value', gt=date_value) nested = Nested(path='dates', query=range) if 'inverted' not in temporal_filter[index]: temporal_filter[index]['inverted'] = False if temporal_filter[index]['inverted']: selectbox_boolfilter.must_not(nested) else: selectbox_boolfilter.must(nested) terms_queries.append(selectbox_boolfilter) # if not selectbox_boolfilter.empty: # if boolean_search == 'or': # boolfilter.should(selectbox_boolfilter) # else: # boolfilter.must(selectbox_boolfilter) # We now have individual query terms for each of the search components. Combine into one group now # Start by building a an array of groups which will be combined according to the global And/Or # Queries within one of these groups will be combined by the complement of the global And/Or # We may end up with [ [A,B], [C], [D,E] ], which would translate to either: # (A || B) && C && (D || E) # or # (A && B) || C || (D && E) # for global AND or OR respectively # logging.warning("TERMS QUERIES %s", terms_queries) bool_components = []; for i, term_query in enumerate(terms_queries): if i is 0: bool_components.append([term_query]) else: should_group_with_previous = filter_combine_flags[i-1] if should_group_with_previous: bool_components[-1].append(term_query) else: bool_components.append([term_query]) # logging.warning("BOOL COMPONENTS %s", bool_components) # Now build the ES queries for bool_component in bool_components: if len(bool_component) is 1: # just combine this on its own q = bool_component[0] else: q = Bool() for sub_component in bool_component: if boolean_search == 'or': #apply the OPPOSITE of the global boolean operator q.must(sub_component) else: q.should(sub_component) # combine to the overall query according to the global boolean operator if boolean_search == 'or': boolfilter.should(q) else: boolfilter.must(q) if 'geometry' in spatial_filter and 'type' in spatial_filter['geometry'] and spatial_filter['geometry']['type'] != '': geojson = spatial_filter['geometry'] if geojson['type'] == 'bbox': coordinates = [[geojson['coordinates'][0],geojson['coordinates'][3]], [geojson['coordinates'][2],geojson['coordinates'][1]]] geoshape = GeoShape(field='geometries.value', type='envelope', coordinates=coordinates ) nested = Nested(path='geometries', query=geoshape) else: buffer = spatial_filter['buffer'] geojson = JSONDeserializer().deserialize(_buffer(geojson,buffer['width'],buffer['unit']).json) geoshape = GeoShape(field='geometries.value', type=geojson['type'], coordinates=geojson['coordinates'] ) nested = Nested(path='geometries', query=geoshape) if 'inverted' not in spatial_filter: spatial_filter['inverted'] = False if spatial_filter['inverted']: boolfilter.must_not(nested) else: boolfilter.must(nested) if not boolquery.empty: query.add_query(boolquery) if not boolfilter.empty: query.add_filter(boolfilter) # Sorting criterion added to query (AZ 10/08/16) query.dsl.update({'sort': sorting}) # logging.warning("-=-==-=-===-=--=-==-=-===-=- query: -=-==-=-===-=--=-==-=-===-=-> %s", query) return query
def search_terms(request): lang = request.GET.get("lang", settings.LANGUAGE_CODE) se = SearchEngineFactory().create() searchString = request.GET.get("q", "") user_is_reviewer = user_is_resource_reviewer(request.user) i = 0 ret = {} for index in ["terms", "concepts"]: query = Query(se, start=0, limit=0) boolquery = Bool() boolquery.should( Match(field="value", query=searchString.lower(), type="phrase_prefix")) boolquery.should( Match(field="value.folded", query=searchString.lower(), type="phrase_prefix")) boolquery.should( Match(field="value.folded", query=searchString.lower(), fuzziness="AUTO", prefix_length=settings.SEARCH_TERM_SENSITIVITY)) if user_is_reviewer is False and index == "terms": boolquery.filter(Terms(field="provisional", terms=["false"])) query.add_query(boolquery) base_agg = Aggregation(name="value_agg", type="terms", field="value.raw", size=settings.SEARCH_DROPDOWN_LENGTH, order={"max_score": "desc"}) nodegroupid_agg = Aggregation(name="nodegroupid", type="terms", field="nodegroupid") top_concept_agg = Aggregation(name="top_concept", type="terms", field="top_concept") conceptid_agg = Aggregation(name="conceptid", type="terms", field="conceptid") max_score_agg = MaxAgg(name="max_score", script="_score") top_concept_agg.add_aggregation(conceptid_agg) base_agg.add_aggregation(max_score_agg) base_agg.add_aggregation(top_concept_agg) base_agg.add_aggregation(nodegroupid_agg) query.add_aggregation(base_agg) ret[index] = [] results = query.search(index=index) if results is not None: for result in results["aggregations"]["value_agg"]["buckets"]: if len(result["top_concept"]["buckets"]) > 0: for top_concept in result["top_concept"]["buckets"]: top_concept_id = top_concept["key"] top_concept_label = get_preflabel_from_conceptid( top_concept["key"], lang)["value"] for concept in top_concept["conceptid"]["buckets"]: ret[index].append({ "type": "concept", "context": top_concept_id, "context_label": top_concept_label, "id": i, "text": result["key"], "value": concept["key"], }) i = i + 1 else: ret[index].append({ "type": "term", "context": "", "context_label": get_resource_model_label(result), "id": i, "text": result["key"], "value": result["key"], }) i = i + 1 return JSONResponse(ret)
def build_search_results_dsl(request): term_filter = request.GET.get('termFilter', '') spatial_filter = JSONDeserializer().deserialize( request.GET.get('mapFilter', '{}')) include_provisional = get_provisional_type(request) export = request.GET.get('export', None) mobile_download = request.GET.get('mobiledownload', None) page = 1 if request.GET.get('page') == '' else int( request.GET.get('page', 1)) temporal_filter = JSONDeserializer().deserialize( request.GET.get('temporalFilter', '{}')) advanced_filters = JSONDeserializer().deserialize( request.GET.get('advanced', '[]')) search_buffer = None se = SearchEngineFactory().create() if export != None: limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE elif mobile_download != None: limit = settings.MOBILE_DOWNLOAD_RESOURCE_LIMIT else: limit = settings.SEARCH_ITEMS_PER_PAGE query = Query(se, start=limit * int(page - 1), limit=limit) search_query = Bool() nested_agg = NestedAgg(path='points', name='geo_aggs') nested_agg_filter = FiltersAgg(name='inner') if include_provisional == True: nested_agg_filter.add_filter( Terms(field='points.provisional', terms=['false', 'true'])) else: provisional_resource_filter = Bool() if include_provisional == False: provisional_resource_filter.filter( Terms(field='provisional', terms=['false', 'partial'])) nested_agg_filter.add_filter( Terms(field='points.provisional', terms=['false'])) elif include_provisional == 'only provisional': provisional_resource_filter.filter( Terms(field='provisional', terms=['true', 'partial'])) nested_agg_filter.add_filter( Terms(field='points.provisional', terms=['true'])) search_query.must(provisional_resource_filter) nested_agg_filter.add_aggregation( GeoHashGridAgg(field='points.point', name='grid', precision=settings.HEX_BIN_PRECISION)) nested_agg_filter.add_aggregation( GeoBoundsAgg(field='points.point', name='bounds')) nested_agg.add_aggregation(nested_agg_filter) query.add_aggregation(nested_agg) permitted_nodegroups = get_permitted_nodegroups(request.user) if term_filter != '': for term in JSONDeserializer().deserialize(term_filter): term_query = Bool() provisional_term_filter = Bool() if term['type'] == 'term' or term['type'] == 'string': string_filter = Bool() if term['type'] == 'term': string_filter.must( Match(field='strings.string', query=term['value'], type='phrase')) elif term['type'] == 'string': string_filter.should( Match(field='strings.string', query=term['value'], type='phrase_prefix')) string_filter.should( Match(field='strings.string.folded', query=term['value'], type='phrase_prefix')) if include_provisional == False: string_filter.must_not( Match(field='strings.provisional', query='true', type='phrase')) elif include_provisional == 'only provisional': string_filter.must_not( Match(field='strings.provisional', query='false', type='phrase')) string_filter.filter( Terms(field='strings.nodegroup_id', terms=permitted_nodegroups)) nested_string_filter = Nested(path='strings', query=string_filter) if term['inverted']: search_query.must_not(nested_string_filter) else: search_query.must(nested_string_filter) # need to set min_score because the query returns results with score 0 and those have to be removed, which I don't think it should be doing query.min_score('0.01') elif term['type'] == 'concept': concept_ids = _get_child_concepts(term['value']) conceptid_filter = Bool() conceptid_filter.filter( Terms(field='domains.conceptid', terms=concept_ids)) conceptid_filter.filter( Terms(field='domains.nodegroup_id', terms=permitted_nodegroups)) if include_provisional == False: conceptid_filter.must_not( Match(field='domains.provisional', query='true', type='phrase')) elif include_provisional == 'only provisional': conceptid_filter.must_not( Match(field='domains.provisional', query='false', type='phrase')) nested_conceptid_filter = Nested(path='domains', query=conceptid_filter) if term['inverted']: search_query.must_not(nested_conceptid_filter) else: search_query.filter(nested_conceptid_filter) if 'features' in spatial_filter: if len(spatial_filter['features']) > 0: feature_geom = spatial_filter['features'][0]['geometry'] feature_properties = {} if 'properties' in spatial_filter['features'][0]: feature_properties = spatial_filter['features'][0][ 'properties'] buffer = {'width': 0, 'unit': 'ft'} if 'buffer' in feature_properties: buffer = feature_properties['buffer'] search_buffer = _buffer(feature_geom, buffer['width'], buffer['unit']) feature_geom = JSONDeserializer().deserialize(search_buffer.json) geoshape = GeoShape(field='geometries.geom.features.geometry', type=feature_geom['type'], coordinates=feature_geom['coordinates']) invert_spatial_search = False if 'inverted' in feature_properties: invert_spatial_search = feature_properties['inverted'] spatial_query = Bool() if invert_spatial_search == True: spatial_query.must_not(geoshape) else: spatial_query.filter(geoshape) # get the nodegroup_ids that the user has permission to search spatial_query.filter( Terms(field='geometries.nodegroup_id', terms=permitted_nodegroups)) if include_provisional == False: spatial_query.filter( Terms(field='geometries.provisional', terms=['false'])) elif include_provisional == 'only provisional': spatial_query.filter( Terms(field='geometries.provisional', terms=['true'])) search_query.filter(Nested(path='geometries', query=spatial_query)) if 'fromDate' in temporal_filter and 'toDate' in temporal_filter: now = str(datetime.utcnow()) start_date = ExtendedDateFormat(temporal_filter['fromDate']) end_date = ExtendedDateFormat(temporal_filter['toDate']) date_nodeid = str( temporal_filter['dateNodeId'] ) if 'dateNodeId' in temporal_filter and temporal_filter[ 'dateNodeId'] != '' else None query_inverted = False if 'inverted' not in temporal_filter else temporal_filter[ 'inverted'] temporal_query = Bool() if query_inverted: # inverted date searches need to use an OR clause and are generally more complicated to structure (can't use ES must_not) # eg: less than START_DATE OR greater than END_DATE inverted_date_query = Bool() inverted_date_ranges_query = Bool() if start_date.is_valid(): inverted_date_query.should( Range(field='dates.date', lt=start_date.lower)) inverted_date_ranges_query.should( Range(field='date_ranges.date_range', lt=start_date.lower)) if end_date.is_valid(): inverted_date_query.should( Range(field='dates.date', gt=end_date.upper)) inverted_date_ranges_query.should( Range(field='date_ranges.date_range', gt=end_date.upper)) date_query = Bool() date_query.filter(inverted_date_query) date_query.filter( Terms(field='dates.nodegroup_id', terms=permitted_nodegroups)) if include_provisional == False: date_query.filter( Terms(field='dates.provisional', terms=['false'])) elif include_provisional == 'only provisional': date_query.filter( Terms(field='dates.provisional', terms=['true'])) if date_nodeid: date_query.filter(Term(field='dates.nodeid', term=date_nodeid)) else: date_ranges_query = Bool() date_ranges_query.filter(inverted_date_ranges_query) date_ranges_query.filter( Terms(field='date_ranges.nodegroup_id', terms=permitted_nodegroups)) if include_provisional == False: date_ranges_query.filter( Terms(field='date_ranges.provisional', terms=['false'])) elif include_provisional == 'only provisional': date_ranges_query.filter( Terms(field='date_ranges.provisional', terms=['true'])) temporal_query.should( Nested(path='date_ranges', query=date_ranges_query)) temporal_query.should(Nested(path='dates', query=date_query)) else: date_query = Bool() date_query.filter( Range(field='dates.date', gte=start_date.lower, lte=end_date.upper)) date_query.filter( Terms(field='dates.nodegroup_id', terms=permitted_nodegroups)) if include_provisional == False: date_query.filter( Terms(field='dates.provisional', terms=['false'])) elif include_provisional == 'only provisional': date_query.filter( Terms(field='dates.provisional', terms=['true'])) if date_nodeid: date_query.filter(Term(field='dates.nodeid', term=date_nodeid)) else: date_ranges_query = Bool() date_ranges_query.filter( Range(field='date_ranges.date_range', gte=start_date.lower, lte=end_date.upper, relation='intersects')) date_ranges_query.filter( Terms(field='date_ranges.nodegroup_id', terms=permitted_nodegroups)) if include_provisional == False: date_ranges_query.filter( Terms(field='date_ranges.provisional', terms=['false'])) if include_provisional == 'only provisional': date_ranges_query.filter( Terms(field='date_ranges.provisional', terms=['true'])) temporal_query.should( Nested(path='date_ranges', query=date_ranges_query)) temporal_query.should(Nested(path='dates', query=date_query)) search_query.filter(temporal_query) datatype_factory = DataTypeFactory() if len(advanced_filters) > 0: advanced_query = Bool() grouped_query = Bool() grouped_queries = [grouped_query] for index, advanced_filter in enumerate(advanced_filters): tile_query = Bool() for key, val in advanced_filter.iteritems(): if key != 'op': node = models.Node.objects.get(pk=key) if request.user.has_perm('read_nodegroup', node.nodegroup): datatype = datatype_factory.get_instance(node.datatype) datatype.append_search_filters(val, node, tile_query, request) nested_query = Nested(path='tiles', query=tile_query) if advanced_filter['op'] == 'or' and index != 0: grouped_query = Bool() grouped_queries.append(grouped_query) grouped_query.must(nested_query) for grouped_query in grouped_queries: advanced_query.should(grouped_query) search_query.must(advanced_query) query.add_query(search_query) if search_buffer != None: search_buffer = search_buffer.geojson return {'query': query, 'search_buffer': search_buffer}
def search_terms(request): lang = request.GET.get('lang', settings.LANGUAGE_CODE) se = SearchEngineFactory().create() searchString = request.GET.get('q', '') query = Query(se, start=0, limit=0) user_is_reviewer = request.user.groups.filter( name='Resource Reviewer').exists() boolquery = Bool() boolquery.should( Match(field='value', query=searchString.lower(), type='phrase_prefix', fuzziness='AUTO')) boolquery.should( Match(field='value.folded', query=searchString.lower(), type='phrase_prefix', fuzziness='AUTO')) boolquery.should( Match(field='value.folded', query=searchString.lower(), fuzziness='AUTO')) if user_is_reviewer is False: boolquery.filter(Terms(field='provisional', terms=['false'])) query.add_query(boolquery) base_agg = Aggregation(name='value_agg', type='terms', field='value.raw', size=settings.SEARCH_DROPDOWN_LENGTH, order={"max_score": "desc"}) nodegroupid_agg = Aggregation(name='nodegroupid', type='terms', field='nodegroupid') top_concept_agg = Aggregation(name='top_concept', type='terms', field='top_concept') conceptid_agg = Aggregation(name='conceptid', type='terms', field='conceptid') max_score_agg = MaxAgg(name='max_score', script='_score') top_concept_agg.add_aggregation(conceptid_agg) base_agg.add_aggregation(max_score_agg) base_agg.add_aggregation(top_concept_agg) base_agg.add_aggregation(nodegroupid_agg) query.add_aggregation(base_agg) results = query.search(index='strings') or {'hits': {'hits': []}} i = 0 ret = [] for result in results['aggregations']['value_agg']['buckets']: if len(result['top_concept']['buckets']) > 0: for top_concept in result['top_concept']['buckets']: top_concept_id = top_concept['key'] top_concept_label = get_preflabel_from_conceptid( top_concept['key'], lang)['value'] for concept in top_concept['conceptid']['buckets']: ret.append({ 'type': 'concept', 'context': top_concept_id, 'context_label': top_concept_label, 'id': i, 'text': result['key'], 'value': concept['key'] }) i = i + 1 else: ret.append({ 'type': 'term', 'context': '', 'context_label': get_resource_model_label(result), 'id': i, 'text': result['key'], 'value': result['key'] }) i = i + 1 return JSONResponse(ret)
def build_base_search_results_dsl(request): term_filter = request.GET.get('termFilter', '') spatial_filter = JSONDeserializer().deserialize(request.GET.get('spatialFilter', None)) export = request.GET.get('export', None) page = 1 if request.GET.get('page') == '' else int(request.GET.get('page', 1)) temporal_filter = JSONDeserializer().deserialize(request.GET.get('temporalFilter', None)) se = SearchEngineFactory().create() if export != None: limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE else: limit = settings.SEARCH_ITEMS_PER_PAGE query = Query(se, start=limit*int(page-1), limit=limit) boolquery = Bool() boolfilter = Bool() if term_filter != '': # Ce uporabnik ni avtenticiran, prikazemo le veljavne (to je verjetno potrebno se dodelati (mogoce da vidijo le svoje???)!!!) if (request.user.username == 'anonymous'): auto_filter = [] for item in JSONDeserializer().deserialize(term_filter): auto_filter.append(item) # Poiscimo concept id in context za Published status AUTO_TERM_FILTER = get_auto_filter(request) auto_filter.append(AUTO_TERM_FILTER) term_filter = JSONSerializer().serialize(auto_filter) print 'term_filter' if term_filter != '': for term in JSONDeserializer().deserialize(term_filter): print term if term['type'] == 'term': entitytype = models.EntityTypes.objects.get(conceptid_id=term['context']) boolfilter_nested = Bool() boolfilter_nested.must(Terms(field='child_entities.entitytypeid', terms=[entitytype.pk])) boolfilter_nested.must(Match(field='child_entities.value', query=term['value'], type='phrase')) nested = Nested(path='child_entities', query=boolfilter_nested) if term['inverted']: boolfilter.must_not(nested) else: boolfilter.must(nested) elif term['type'] == 'concept': concept_ids = _get_child_concepts(term['value']) terms = Terms(field='domains.conceptid', terms=concept_ids) nested = Nested(path='domains', query=terms) if term['inverted']: boolfilter.must_not(nested) else: boolfilter.must(nested) elif term['type'] == 'string': boolfilter_folded = Bool() boolfilter_folded.should(Match(field='child_entities.value', query=term['value'], type='phrase_prefix')) boolfilter_folded.should(Match(field='child_entities.value.folded', query=term['value'], type='phrase_prefix')) nested = Nested(path='child_entities', query=boolfilter_folded) if term['inverted']: boolquery.must_not(nested) else: boolquery.must(nested) if 'geometry' in spatial_filter and 'type' in spatial_filter['geometry'] and spatial_filter['geometry']['type'] != '': geojson = spatial_filter['geometry'] if geojson['type'] == 'bbox': coordinates = [[geojson['coordinates'][0],geojson['coordinates'][3]], [geojson['coordinates'][2],geojson['coordinates'][1]]] geoshape = GeoShape(field='geometries.value', type='envelope', coordinates=coordinates ) nested = Nested(path='geometries', query=geoshape) else: buffer = spatial_filter['buffer'] geojson = JSONDeserializer().deserialize(_buffer(geojson,buffer['width'],buffer['unit']).json) geoshape = GeoShape(field='geometries.value', type=geojson['type'], coordinates=geojson['coordinates'] ) nested = Nested(path='geometries', query=geoshape) if 'inverted' not in spatial_filter: spatial_filter['inverted'] = False if spatial_filter['inverted']: boolfilter.must_not(nested) else: boolfilter.must(nested) if 'year_min_max' in temporal_filter and len(temporal_filter['year_min_max']) == 2: start_date = date(temporal_filter['year_min_max'][0], 1, 1) end_date = date(temporal_filter['year_min_max'][1], 12, 31) if start_date: start_date = start_date.isoformat() if end_date: end_date = end_date.isoformat() range = Range(field='dates.value', gte=start_date, lte=end_date) nested = Nested(path='dates', query=range) if 'inverted' not in temporal_filter: temporal_filter['inverted'] = False if temporal_filter['inverted']: boolfilter.must_not(nested) else: boolfilter.must(nested) if not boolquery.empty: query.add_query(boolquery) if not boolfilter.empty: query.add_filter(boolfilter) return query
def build_search_results_dsl(request): term_filter = request.GET.get('termFilter', '') spatial_filter = JSONDeserializer().deserialize( request.GET.get('mapFilter', '{}')) export = request.GET.get('export', None) page = 1 if request.GET.get('page') == '' else int( request.GET.get('page', 1)) temporal_filter = JSONDeserializer().deserialize( request.GET.get('temporalFilter', '{}')) se = SearchEngineFactory().create() if export != None: limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE else: limit = settings.SEARCH_ITEMS_PER_PAGE query = Query(se, start=limit * int(page - 1), limit=limit) boolquery = Bool() boolfilter = Bool() if term_filter != '': for term in JSONDeserializer().deserialize(term_filter): if term['type'] == 'term': term_filter = Bool() term_filter.must( Match(field='strings', query=term['value'], type='phrase')) if term['inverted']: boolfilter.must_not(term_filter) else: boolfilter.must(term_filter) elif term['type'] == 'concept': concept_ids = _get_child_concepts(term['value']) conceptid_filter = Terms(field='domains.conceptid', terms=concept_ids) if term['inverted']: boolfilter.must_not(conceptid_filter) else: boolfilter.must(conceptid_filter) elif term['type'] == 'string': string_filter = Bool() string_filter.should( Match(field='strings', query=term['value'], type='phrase_prefix')) string_filter.should( Match(field='strings.folded', query=term['value'], type='phrase_prefix')) if term['inverted']: boolfilter.must_not(string_filter) else: boolfilter.must(string_filter) if 'features' in spatial_filter: if len(spatial_filter['features']) > 0: feature_geom = spatial_filter['features'][0]['geometry'] feature_properties = spatial_filter['features'][0]['properties'] buffer = {'width': 0, 'unit': 'ft'} if 'buffer' in feature_properties: buffer = feature_properties['buffer'] feature_geom = JSONDeserializer().deserialize( _buffer(feature_geom, buffer['width'], buffer['unit']).json) geoshape = GeoShape(field='geometries.features.geometry', type=feature_geom['type'], coordinates=feature_geom['coordinates']) invert_spatial_search = False if 'inverted' in feature_properties: invert_spatial_search = feature_properties['inverted'] if invert_spatial_search == True: boolfilter.must_not(geoshape) else: boolfilter.must(geoshape) if 'fromDate' in temporal_filter and 'toDate' in temporal_filter: start_date = None end_date = None try: start_date = parser.parse(temporal_filter['fromDate']) start_date = start_date.isoformat() except: pass try: end_date = parser.parse(temporal_filter['toDate']) end_date = end_date.isoformat() except: pass if 'dateNodeId' in temporal_filter and temporal_filter[ 'dateNodeId'] != '': range = Range(field='tiles.data.%s' % (temporal_filter['dateNodeId']), gte=start_date, lte=end_date) time_query_dsl = Nested(path='tiles', query=range) else: time_query_dsl = Range(field='dates', gte=start_date, lte=end_date) if 'inverted' not in temporal_filter: temporal_filter['inverted'] = False if temporal_filter['inverted']: boolfilter.must_not(time_query_dsl) else: boolfilter.must(time_query_dsl) if not boolquery.empty: query.add_query(boolquery) if not boolfilter.empty: query.add_filter(boolfilter) return query
def append_dsl(self, search_results_object, permitted_nodegroups, include_provisional): search_query = Bool() querysting_params = self.request.GET.get(details["componentname"], "") for term in JSONDeserializer().deserialize(querysting_params): if term["type"] == "term" or term["type"] == "string": string_filter = Bool() if term["type"] == "term": string_filter.must( Match(field="strings.string", query=term["value"], type="phrase")) elif term["type"] == "string": string_filter.should( Match(field="strings.string", query=term["value"], type="phrase_prefix")) string_filter.should( Match(field="strings.string.folded", query=term["value"], type="phrase_prefix")) if include_provisional is False: string_filter.must_not( Match(field="strings.provisional", query="true", type="phrase")) elif include_provisional == "only provisional": string_filter.must_not( Match(field="strings.provisional", query="false", type="phrase")) string_filter.filter( Terms(field="strings.nodegroup_id", terms=permitted_nodegroups)) nested_string_filter = Nested(path="strings", query=string_filter) if term["inverted"]: search_query.must_not(nested_string_filter) else: search_query.must(nested_string_filter) # need to set min_score because the query returns results with score 0 and those have to be removed, which I don't think it should be doing search_results_object["query"].min_score("0.01") elif term["type"] == "concept": concept_ids = _get_child_concepts(term["value"]) conceptid_filter = Bool() conceptid_filter.filter( Terms(field="domains.conceptid", terms=concept_ids)) conceptid_filter.filter( Terms(field="domains.nodegroup_id", terms=permitted_nodegroups)) if include_provisional is False: conceptid_filter.must_not( Match(field="domains.provisional", query="true", type="phrase")) elif include_provisional == "only provisional": conceptid_filter.must_not( Match(field="domains.provisional", query="false", type="phrase")) nested_conceptid_filter = Nested(path="domains", query=conceptid_filter) if term["inverted"]: search_query.must_not(nested_conceptid_filter) else: search_query.filter(nested_conceptid_filter) search_results_object["query"].add_query(search_query)
def nested_query_from_pathed_values(pathed_values, stem): """ Given an array of pathed values to query terms from the root, return a nested query pathed_values: e.g. [ { val: '29430-4955-...' path: [a, b, c] } ] stem: the path into the index for the nested terms. This will be of the form 'nested_entity.child_entities.child_entities' """ # f( [[A,B,C], [A,B,D] ) # = Nested( AND( f( [[B,C],[B,D]] )) # = Nested( AND( Nested( AND( f([[C],[D]]) )) )) # = Nested( AND( Nested( AND( valueC, valueD)))) # f( [[A,B,C], [A,B,D], [A,B,D] ) # = Nested( AND( f([[B,C],[B,D],[B,D]] )) # = Nested( AND( Nested( AND( f([[C],[D],[D]]) )) )) # = Nested( AND( Nested( AND( valueB, valueD)))) # group paths by their head of each paths list is the same, make a single nested query and recurse on the tails branch_groups = {} # those groups with a continuing tail, where we will recursively build a nested query leaf_groups = [] # those groups without a continuing tail, where we will use an ordinary term query # build the groups for v in pathed_values: path = v['path'] if len(path) == 1: # this goes in its own group leaf_groups.append(v) else: # see if there is already a group using this head head = v['path'][0]['entitytyperange'] if head not in branch_groups: branch_groups[head] = [] branch_groups[head].append(v) # We should now have a set of groups # create the bool query bool_term = Bool() # add terms for any leaf groups for leaf_group in leaf_groups: if leaf_group['term']['type'] == 'concept': if leaf_group['term']['inverted']: terms = Terms(field=stem+'.conceptid', terms=leaf_group['term']['value']) n_terms = Nested(path=stem, query=terms) bool_term.must_not(n_terms) else: terms = Terms(field=stem+'.conceptid', terms=leaf_group['term']['value']) n_terms = Nested(path=stem, query=terms) bool_term.must(n_terms) elif leaf_group['term']['type'] == 'term': # boolfilter_nested.must(Terms(field='child_entities.entitytypeid', terms=[entitytype.pk])) # boolfilter_nested.must(Match(field='child_entities.value', query=term['value'], type='phrase')) entitytype = models.EntityTypes.objects.get(conceptid_id=leaf_group['term']['context']) sub_bool = Bool() if leaf_group['term']['inverted']: sub_bool.must_not(Terms(field=stem+'.entitytypeid', terms=[entitytype.pk])) sub_bool.must_not(Match(field=stem+'.value', query=leaf_group['term']['value'], type='phrase')) else: sub_bool.must(Terms(field=stem+'.entitytypeid', terms=[entitytype.pk])) sub_bool.must(Match(field=stem+'.value', query=leaf_group['term']['value'], type='phrase')) nsub_bool = Nested(path=stem, query=sub_bool) bool_term.must(nsub_bool) elif leaf_group['term']['type'] == 'string': boolfilter_folded = Bool() boolfilter_folded.should(Match(field=stem+'.flat_child_entities.label', query=leaf_group['term']['value'], type='phrase_prefix', fuzziness='AUTO')) boolfilter_folded.should(Match(field=stem+'.flat_child_entities.label.folded', query=leaf_group['term']['value'], type='phrase_prefix', fuzziness='AUTO')) boolfilter_folded.should(Match(field=stem+'.flat_child_entities.label.folded', query=leaf_group['term']['value'], fuzziness='AUTO')) nested = Nested(path=stem+'.flat_child_entities', query=boolfilter_folded) if leaf_group['term']['inverted']: bool_term.must_not(nested) else: bool_term.must(nested) elif leaf_group['term']['type'] == 'date': if leaf_group['term']['date_operator'] == '1': # equals query daterange = Range(field=stem+'.flat_child_entities.date', gte=leaf_group['term']['date_value'], lte=leaf_group['term']['date_value']) elif leaf_group['term']['date_operator'] == '0': # greater than query daterange = Range(field=stem+'.flat_child_entities.date', lt=leaf_group['term']['date_value']) elif leaf_group['term']['date_operator'] == '2': # less than query daterange = Range(field=stem+'.flat_child_entities.date', gt=leaf_group['term']['date_value']) elif leaf_group['term']['date_operator'] == '3': # greater than and less than query daterange = Range(field=stem+'.flat_child_entities.date', gte=leaf_group['term']['start_date'], lte=leaf_group['term']['end_date']) nested_date = Nested(path=stem+'.flat_child_entities', query=daterange) if leaf_group['term']['inverted']: bool_term.must_not(nested_date) else: bool_term.must(nested_date) # add terms for any branch groups for key in branch_groups: # add a nested term for each group branch_group = branch_groups[key] #remove head from each path and recurse for value in branch_group: value['path'] = value['path'][1:] sub_query = nested_query_from_pathed_values(branch_group, stem+'.child_entities') nsub_query = Nested(path=stem, query=sub_query) bool_term.must(nsub_query) return bool_term;
def search(request): searchString = request.GET["q"] removechildren = request.GET.get("removechildren", None) query = Query(se, start=0, limit=100) phrase = Match(field="value", query=searchString.lower(), type="phrase_prefix") query.add_query(phrase) results = query.search(index=CONCEPTS_INDEX) ids = [] if removechildren is not None: ids = [ concept[0] for concept in Concept().get_child_concepts( removechildren, columns="conceptidto::text") ] ids.append(removechildren) newresults = [] cached_scheme_names = {} for result in results["hits"]["hits"]: if result["_source"]["conceptid"] not in ids: # first look to see if we've already retrieved the top concept name # else look up the top concept name with ES and cache the result top_concept = result["_source"]["top_concept"] if top_concept in cached_scheme_names: result["in_scheme_name"] = cached_scheme_names[top_concept] else: query = Query(se, start=0, limit=100) phrase = Match(field="conceptid", query=top_concept, type="phrase") query.add_query(phrase) scheme = query.search(index=CONCEPTS_INDEX) for label in scheme["hits"]["hits"]: if label["_source"]["type"] == "prefLabel": cached_scheme_names[top_concept] = label["_source"][ "value"] result["in_scheme_name"] = label["_source"]["value"] newresults.append(result) # Use the db to get the concept context but this is SLOW # for result in results['hits']['hits']: # if result['_source']['conceptid'] not in ids: # concept = Concept().get(id=result['_source']['conceptid'], include_parentconcepts=True) # pathlist = concept.get_paths() # result['in_scheme_name'] = pathlist[0][0]['label'] # newresults.append(result) # def crawl(conceptid, path=[]): # query = Query(se, start=0, limit=100) # bool = Bool() # bool.must(Match(field='conceptto', query=conceptid, type='phrase')) # bool.must(Match(field='relationtype', query='narrower', type='phrase')) # query.add_query(bool) # relations = query.search(index='concept_relations') # for relation in relations['hits']['hits']: # path.insert(0, relation) # crawl(relation['_source']['conceptfrom'], path=path) # return path # for result in results['hits']['hits']: # if result['_source']['conceptid'] not in ids: # concept_relations = crawl(result['_source']['conceptid'], path=[]) # if len(concept_relations) > 0: # conceptid = concept_relations[0]['_source']['conceptfrom'] # if conceptid in cached_scheme_names: # result['in_scheme_name'] = cached_scheme_names[conceptid] # else: # result['in_scheme_name'] = get_preflabel_from_conceptid(conceptid, lang=request.LANGUAGE_CODE)['value'] # cached_scheme_names[conceptid] = result['in_scheme_name'] # newresults.append(result) results["hits"]["hits"] = newresults return JSONResponse(results)
def search(request): se = SearchEngineFactory().create() searchString = request.GET['q'] removechildren = request.GET.get('removechildren', None) query = Query(se, start=0, limit=100) phrase = Match(field='value', query=searchString.lower(), type='phrase_prefix') query.add_query(phrase) results = query.search(index='concept_labels') ids = [] if removechildren != None: concepts = Concept().get(id=removechildren, include_subconcepts=True, include=None) def get_children(concept): ids.append(concept.id) concepts.traverse(get_children) newresults = [] cached_scheme_names = {} for result in results['hits']['hits']: if result['_source']['conceptid'] not in ids: # first look to see if we've already retrieved the scheme name # else look up the scheme name with ES and cache the result if result['_type'] in cached_scheme_names: result['in_scheme_name'] = cached_scheme_names[result['_type']] else: query = Query(se, start=0, limit=100) phrase = Match(field='conceptid', query=result['_type'], type='phrase') query.add_query(phrase) scheme = query.search(index='concept_labels') for label in scheme['hits']['hits']: if label['_source']['type'] == 'prefLabel': cached_scheme_names[ result['_type']] = label['_source']['value'] result['in_scheme_name'] = label['_source']['value'] newresults.append(result) # Use the db to get the concept context but this is SLOW # for result in results['hits']['hits']: # if result['_source']['conceptid'] not in ids: # concept = Concept().get(id=result['_source']['conceptid'], include_parentconcepts=True) # pathlist = concept.get_paths() # result['in_scheme_name'] = pathlist[0][0]['label'] # newresults.append(result) # def crawl(conceptid, path=[]): # query = Query(se, start=0, limit=100) # bool = Bool() # bool.must(Match(field='conceptidto', query=conceptid, type='phrase')) # bool.must(Match(field='relationtype', query='narrower', type='phrase')) # query.add_query(bool) # relations = query.search(index='concept_relations') # for relation in relations['hits']['hits']: # path.insert(0, relation) # crawl(relation['_source']['conceptidfrom'], path=path) # return path # for result in results['hits']['hits']: # if result['_source']['conceptid'] not in ids: # concept_relations = crawl(result['_source']['conceptid'], path=[]) # if len(concept_relations) > 0: # conceptid = concept_relations[0]['_source']['conceptidfrom'] # if conceptid in cached_scheme_names: # result['in_scheme_name'] = cached_scheme_names[conceptid] # else: # result['in_scheme_name'] = get_preflabel_from_conceptid(conceptid, lang=settings.LANGUAGE_CODE)['value'] # cached_scheme_names[conceptid] = result['in_scheme_name'] # newresults.append(result) results['hits']['hits'] = newresults return JSONResponse(results)
def append_dsl(self, search_results_object, permitted_nodegroups, include_provisional): search_query = Bool() querysting_params = self.request.GET.get(details['componentname'], '') for term in JSONDeserializer().deserialize(querysting_params): if term['type'] == 'term' or term['type'] == 'string': string_filter = Bool() if term['type'] == 'term': string_filter.must( Match(field='strings.string', query=term['value'], type='phrase')) elif term['type'] == 'string': string_filter.should( Match(field='strings.string', query=term['value'], type='phrase_prefix')) string_filter.should( Match(field='strings.string.folded', query=term['value'], type='phrase_prefix')) if include_provisional is False: string_filter.must_not( Match(field='strings.provisional', query='true', type='phrase')) elif include_provisional == 'only provisional': string_filter.must_not( Match(field='strings.provisional', query='false', type='phrase')) string_filter.filter( Terms(field='strings.nodegroup_id', terms=permitted_nodegroups)) nested_string_filter = Nested(path='strings', query=string_filter) if term['inverted']: search_query.must_not(nested_string_filter) else: search_query.must(nested_string_filter) # need to set min_score because the query returns results with score 0 and those have to be removed, which I don't think it should be doing search_results_object['query'].min_score('0.01') elif term['type'] == 'concept': concept_ids = _get_child_concepts(term['value']) conceptid_filter = Bool() conceptid_filter.filter( Terms(field='domains.conceptid', terms=concept_ids)) conceptid_filter.filter( Terms(field='domains.nodegroup_id', terms=permitted_nodegroups)) if include_provisional is False: conceptid_filter.must_not( Match(field='domains.provisional', query='true', type='phrase')) elif include_provisional == 'only provisional': conceptid_filter.must_not( Match(field='domains.provisional', query='false', type='phrase')) nested_conceptid_filter = Nested(path='domains', query=conceptid_filter) if term['inverted']: search_query.must_not(nested_conceptid_filter) else: search_query.filter(nested_conceptid_filter) search_results_object['query'].add_query(search_query)
def build_search_results_dsl(request): term_filter = request.GET.get('termFilter', '') spatial_filter = JSONDeserializer().deserialize(request.GET.get('spatialFilter', None)) export = request.GET.get('export', None) page = 1 if request.GET.get('page') == '' else int(request.GET.get('page', 1)) temporal_filter = JSONDeserializer().deserialize(request.GET.get('temporalFilter', None)) se = SearchEngineFactory().create() if export != None: limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE else: limit = settings.SEARCH_ITEMS_PER_PAGE query = Query(se, start=limit*int(page-1), limit=limit) boolquery = Bool() boolfilter = Bool() if term_filter != '': for term in JSONDeserializer().deserialize(term_filter): if term['type'] == 'term': entitytype = models.EntityTypes.objects.get(conceptid_id=term['context']) boolfilter_nested = Bool() boolfilter_nested.must(Terms(field='child_entities.entitytypeid', terms=[entitytype.pk])) boolfilter_nested.must(Match(field='child_entities.value', query=term['value'], type='phrase')) nested = Nested(path='child_entities', query=boolfilter_nested) if term['inverted']: boolfilter.must_not(nested) else: boolfilter.must(nested) elif term['type'] == 'concept': concept_ids = _get_child_concepts(term['value']) terms = Terms(field='domains.conceptid', terms=concept_ids) nested = Nested(path='domains', query=terms) if term['inverted']: boolfilter.must_not(nested) else: boolfilter.must(nested) elif term['type'] == 'string': boolfilter_folded = Bool() boolfilter_folded.should(Match(field='child_entities.value', query=term['value'], type='phrase_prefix')) boolfilter_folded.should(Match(field='child_entities.value.folded', query=term['value'], type='phrase_prefix')) nested = Nested(path='child_entities', query=boolfilter_folded) if term['inverted']: boolquery.must_not(nested) else: boolquery.must(nested) if 'geometry' in spatial_filter and 'type' in spatial_filter['geometry'] and spatial_filter['geometry']['type'] != '': geojson = spatial_filter['geometry'] if geojson['type'] == 'bbox': coordinates = [[geojson['coordinates'][0],geojson['coordinates'][3]], [geojson['coordinates'][2],geojson['coordinates'][1]]] geoshape = GeoShape(field='geometries.value', type='envelope', coordinates=coordinates ) nested = Nested(path='geometries', query=geoshape) else: buffer = spatial_filter['buffer'] geojson = JSONDeserializer().deserialize(_buffer(geojson,buffer['width'],buffer['unit']).json) geoshape = GeoShape(field='geometries.value', type=geojson['type'], coordinates=geojson['coordinates'] ) nested = Nested(path='geometries', query=geoshape) if 'inverted' not in spatial_filter: spatial_filter['inverted'] = False if spatial_filter['inverted']: boolfilter.must_not(nested) else: boolfilter.must(nested) if 'year_min_max' in temporal_filter and len(temporal_filter['year_min_max']) == 2: start = temporal_filter['year_min_max'][0]*10000 end = temporal_filter['year_min_max'][1]*10000 range = Range(field='extendeddates.value', gte=start, lte=end) nested = Nested(path='extendeddates', query=range) if 'inverted' not in temporal_filter: temporal_filter['inverted'] = False if temporal_filter['inverted']: boolfilter.must_not(nested) else: boolfilter.must(nested) if 'filters' in temporal_filter: time_boolfilter = Bool() for temporal_filter in temporal_filter['filters']: date_type = '' date = '' date_operator = '' for node in temporal_filter['nodes']: if node['entitytypeid'] == 'DATE_COMPARISON_OPERATOR.E55': date_operator = node['value'] elif node['entitytypeid'] == 'date': date = node['value'] else: date_type = node['value'] terms = Terms(field='extendeddategroups.conceptid', terms=date_type) boolfilter.must(terms) date_value = date_to_int(date) if date_operator == '1': # equals query range = Range(field='extendeddategroups.value', gte=date_value, lte=date_value) elif date_operator == '0': # greater than query range = Range(field='extendeddategroups.value', lt=date_value) elif date_operator == '2': # less than query range = Range(field='extendeddategroups.value', gt=date_value) if 'inverted' not in temporal_filter: temporal_filter['inverted'] = False if temporal_filter['inverted']: boolfilter.must_not(range) else: boolfilter.must(range) if not boolquery.empty: query.add_query(boolquery) if not boolfilter.empty: query.add_filter(boolfilter) return query
def build_search_results_dsl(request): term_filter = request.GET.get('termFilter', '') spatial_filter = JSONDeserializer().deserialize( request.GET.get('mapFilter', '{}')) export = request.GET.get('export', None) page = 1 if request.GET.get('page') == '' else int( request.GET.get('page', 1)) temporal_filter = JSONDeserializer().deserialize( request.GET.get('temporalFilter', '{}')) se = SearchEngineFactory().create() if export != None: limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE else: limit = settings.SEARCH_ITEMS_PER_PAGE query = Query(se, start=limit * int(page - 1), limit=limit) query.add_aggregation( GeoHashGridAgg(field='points', name='grid', precision=settings.HEX_BIN_PRECISION)) query.add_aggregation(GeoBoundsAgg(field='points', name='bounds')) search_query = Bool() if term_filter != '': for term in JSONDeserializer().deserialize(term_filter): if term['type'] == 'term': term_filter = Match(field='strings', query=term['value'], type='phrase') if term['inverted']: search_query.must_not(term_filter) else: search_query.must(term_filter) elif term['type'] == 'concept': concept_ids = _get_child_concepts(term['value']) conceptid_filter = Terms(field='domains.conceptid', terms=concept_ids) if term['inverted']: search_query.must_not(conceptid_filter) else: search_query.must(conceptid_filter) elif term['type'] == 'string': string_filter = Bool() string_filter.should( Match(field='strings', query=term['value'], type='phrase_prefix')) string_filter.should( Match(field='strings.folded', query=term['value'], type='phrase_prefix')) if term['inverted']: search_query.must_not(string_filter) else: search_query.must(string_filter) if 'features' in spatial_filter: if len(spatial_filter['features']) > 0: feature_geom = spatial_filter['features'][0]['geometry'] feature_properties = spatial_filter['features'][0]['properties'] buffer = {'width': 0, 'unit': 'ft'} if 'buffer' in feature_properties: buffer = feature_properties['buffer'] feature_geom = JSONDeserializer().deserialize( _buffer(feature_geom, buffer['width'], buffer['unit']).json) geoshape = GeoShape(field='geometries.features.geometry', type=feature_geom['type'], coordinates=feature_geom['coordinates']) invert_spatial_search = False if 'inverted' in feature_properties: invert_spatial_search = feature_properties['inverted'] if invert_spatial_search == True: search_query.must_not(geoshape) else: search_query.must(geoshape) if 'fromDate' in temporal_filter and 'toDate' in temporal_filter: now = str(datetime.utcnow()) start_date = None end_date = None start_year = 'null' end_year = 'null' try: # start_date = parser.parse(temporal_filter['fromDate']) # start_date = start_date.isoformat() sd = FlexiDate.from_str(temporal_filter['fromDate']) start_date = int((sd.as_float() - 1970) * 31556952 * 1000) #start_year = parser.parse(start_date).year start_year = sd.year except: pass try: # end_date = parser.parse(temporal_filter['toDate']) # end_date = end_date.isoformat() ed = FlexiDate.from_str(temporal_filter['toDate']) end_date = int((ed.as_float() - 1970) * 31556952 * 1000) #end_year = parser.parse(end_date).year end_year = ed.year except: pass # add filter for concepts that define min or max dates sql = None basesql = """ SELECT value.conceptid FROM ( SELECT {select_clause}, v.conceptid FROM public."values" v, public."values" v2 WHERE v.conceptid = v2.conceptid and v.valuetype = 'min_year' and v2.valuetype = 'max_year' ) as value WHERE overlap = true; """ temporal_query = Bool() if 'inverted' not in temporal_filter: temporal_filter['inverted'] = False if temporal_filter['inverted']: # inverted date searches need to use an OR clause and are generally more complicated to structure (can't use ES must_not) # eg: less than START_DATE OR greater than END_DATE select_clause = [] inverted_date_filter = Bool() field = 'dates' if 'dateNodeId' in temporal_filter and temporal_filter[ 'dateNodeId'] != '': field = 'tiles.data.%s' % (temporal_filter['dateNodeId']) if start_date is not None: inverted_date_filter.should(Range(field=field, lte=start_date)) select_clause.append( "(numrange(v.value::int, v2.value::int, '[]') && numrange(null,{start_year},'[]'))" ) if end_date is not None: inverted_date_filter.should(Range(field=field, gte=end_date)) select_clause.append( "(numrange(v.value::int, v2.value::int, '[]') && numrange({end_year},null,'[]'))" ) if 'dateNodeId' in temporal_filter and temporal_filter[ 'dateNodeId'] != '': date_range_query = Nested(path='tiles', query=inverted_date_filter) temporal_query.should(date_range_query) else: temporal_query.should(inverted_date_filter) select_clause = " or ".join(select_clause) + " as overlap" sql = basesql.format(select_clause=select_clause).format( start_year=start_year, end_year=end_year) else: if 'dateNodeId' in temporal_filter and temporal_filter[ 'dateNodeId'] != '': range = Range(field='tiles.data.%s' % (temporal_filter['dateNodeId']), gte=start_date, lte=end_date) date_range_query = Nested(path='tiles', query=range) temporal_query.should(date_range_query) else: date_range_query = Range(field='dates', gte=start_date, lte=end_date) temporal_query.should(date_range_query) select_clause = """ numrange(v.value::int, v2.value::int, '[]') && numrange({start_year},{end_year},'[]') as overlap """ sql = basesql.format(select_clause=select_clause).format( start_year=start_year, end_year=end_year) # is a dateNodeId is not specified if sql is not None: cursor = connection.cursor() cursor.execute(sql) ret = [str(row[0]) for row in cursor.fetchall()] if len(ret) > 0: conceptid_filter = Terms(field='domains.conceptid', terms=ret) temporal_query.should(conceptid_filter) search_query.must(temporal_query) query.add_query(search_query) return query