def get_related_resources(resourceid, lang, limit=1000, start=0):
ret = {
'resource_relationships': [],
'related_resources': []
}
se = SearchEngineFactory().create()
query = Query(se, limit=limit, start=start)
query.add_filter(Terms(field='entityid1', terms=resourceid).dsl, operator='or')
query.add_filter(Terms(field='entityid2', terms=resourceid).dsl, operator='or')
resource_relations = query.search(index='resource_relations', doc_type='all')
ret['total'] = resource_relations['hits']['total']
entityids = set()
for relation in resource_relations['hits']['hits']:
relation['_source']['preflabel'] = get_preflabel_from_valueid(relation['_source']['relationshiptype'], lang)
ret['resource_relationships'].append(relation['_source'])
entityids.add(relation['_source']['entityid1'])
entityids.add(relation['_source']['entityid2'])
if len(entityids) > 0:
entityids.remove(resourceid)
related_resources = se.search(index='entity', doc_type='_all', id=list(entityids))
if related_resources:
for resource in related_resources['docs']:
ret['related_resources'].append(resource['_source'])
return ret
def get_preflabel_from_conceptid(conceptid, lang):
ret = None
default = {
"category": "",
"conceptid": "",
"language": "",
"value": "",
"type": "",
"id": ""
}
se = SearchEngineFactory().create()
query = Query(se)
terms = Terms(field='conceptid', terms=[conceptid])
match = Match(field='type', query='preflabel', type='phrase')
query.add_filter(terms)
query.add_query(match)
preflabels = query.search(index='concept_labels')['hits']['hits']
for preflabel in preflabels:
default = preflabel['_source']
# get the label in the preferred language, otherwise get the label in the default language
if preflabel['_source']['language'] == lang:
return preflabel['_source']
if preflabel['_source']['language'].split('-')[0] == lang.split(
'-')[0]:
ret = preflabel['_source']
if preflabel['_source'][
'language'] == settings.LANGUAGE_CODE and ret == None:
ret = preflabel['_source']
return default if ret == None else ret
def get_preflabel_from_conceptid(conceptid, lang):
ret = None
default = {
"category": "",
"conceptid": "",
"language": "",
"value": "",
"type": "",
"id": ""
}
se = SearchEngineFactory().create()
query = Query(se)
terms = Terms(field='conceptid', terms=[conceptid])
# Uncomment the following line only after having reindexed ElasticSearch cause currently the Arabic labels are indexed as altLabels
# match = Match(field='type', query='prefLabel', type='phrase')
query.add_filter(terms)
# Uncomment the following line only after having reindexed ElasticSearch cause currently the Arabic labels are indexed as altLabels
# query.add_query(match)
preflabels = query.search(index='concept_labels')['hits']['hits']
for preflabel in preflabels:
# print 'Language at this point %s and label language %s and ret is %s' % (lang, preflabel['_source']['language'], ret)
default = preflabel['_source']
# get the label in the preferred language, otherwise get the label in the default language
if preflabel['_source']['language'] == lang:
# print 'prefLabel from Conceptid: %s' % preflabel['_source']
return preflabel['_source']
if preflabel['_source']['language'].split('-')[0] == lang.split(
'-')[0]:
ret = preflabel['_source']
if preflabel['_source']['language'] == lang and ret == None:
ret = preflabel['_source']
return default if ret == None else ret
def get_related_resources(resourceid, lang, limit=1000, start=0):
ret = {'resource_relationships': [], 'related_resources': []}
se = SearchEngineFactory().create()
query = Query(se, limit=limit, start=start)
query.add_filter(Terms(field='entityid1', terms=resourceid).dsl,
operator='or')
query.add_filter(Terms(field='entityid2', terms=resourceid).dsl,
operator='or')
resource_relations = query.search(index='resource_relations',
doc_type='all')
ret['total'] = resource_relations['hits']['total']
entityids = set()
for relation in resource_relations['hits']['hits']:
relation['_source']['preflabel'] = get_preflabel_from_valueid(
relation['_source']['relationshiptype'], lang)
ret['resource_relationships'].append(relation['_source'])
entityids.add(relation['_source']['entityid1'])
entityids.add(relation['_source']['entityid2'])
if len(entityids) > 0:
entityids.remove(resourceid)
related_resources = se.search(index='entity',
doc_type='_all',
id=list(entityids))
if related_resources:
for resource in related_resources['docs']:
ret['related_resources'].append(resource['_source'])
return ret
def get_preflabel_from_conceptid(conceptid, lang):
ret = None
default = {
"category": "",
"conceptid": "",
"language": "",
"value": "",
"type": "",
"id": ""
}
se = SearchEngineFactory().create()
query = Query(se)
terms = Terms(field='conceptid', terms=[conceptid])
# Uncomment the following line only after having reindexed ElasticSearch cause currently the Arabic labels are indexed as altLabels
# match = Match(field='type', query='prefLabel', type='phrase')
query.add_filter(terms)
# Uncomment the following line only after having reindexed ElasticSearch cause currently the Arabic labels are indexed as altLabels
# query.add_query(match)
preflabels = query.search(index='concept_labels')['hits']['hits']
for preflabel in preflabels:
# print 'Language at this point %s and label language %s and ret is %s' % (lang, preflabel['_source']['language'], ret)
default = preflabel['_source']
# get the label in the preferred language, otherwise get the label in the default language
if preflabel['_source']['language'] == lang:
# print 'prefLabel from Conceptid: %s' % preflabel['_source']
return preflabel['_source']
if preflabel['_source']['language'].split('-')[0] == lang.split('-')[0]:
ret = preflabel['_source']
if preflabel['_source']['language'] == lang and ret == None:
ret = preflabel['_source']
return default if ret == None else ret
def find_overlapping(request):
'''This function queries ES when called via Ajax when a new geometry is created in the Location tab. If pre-existing resources are found within the perimeter of the polygon (or the buffered zone around a point/line/polygon), an alert is raised.'''
geomString = request.GET.get('geom', '')
geom = GEOSGeometry(geomString, srid=4326)
mindistance = settings.METER_RADIUS
if not mindistance:
mindistance = 1000 # if settings.METER_RADIUS isn't set, default to 1Km
geom.transform(3857)
buffered_geom = geom.buffer(mindistance)
buffered_geom.transform(4326)
print geom, buffered_geom
se = SearchEngineFactory().create()
query = Query(se)
boolfilter = Bool()
geoshape = GeoShape(field='geometries.value',
type=buffered_geom.geom_type,
coordinates=buffered_geom.coords)
nested = Nested(path='geometries', query=geoshape)
boolfilter.must(nested)
query.add_filter(boolfilter)
results = query.search(index='entity', doc_type='')
overlaps = []
for hit in results['hits']['hits']:
overlaps.append({
'id': hit['_id'],
'type': hit['_type'],
'primaryname': hit['_source']['primaryname']
})
return JSONResponse(overlaps)
def get_preflabel_from_conceptid(conceptid, lang):
ret = None
default = {
"category": "",
"conceptid": "",
"language": "",
"value": "",
"type": "",
"id": ""
}
se = SearchEngineFactory().create()
query = Query(se)
terms = Terms(field='conceptid', terms=[conceptid])
match = Match(field='type', query='preflabel', type='phrase')
query.add_filter(terms)
query.add_query(match)
preflabels = query.search(index='concept_labels')['hits']['hits']
for preflabel in preflabels:
default = preflabel['_source']
# get the label in the preferred language, otherwise get the label in the default language
if preflabel['_source']['language'] == lang:
return preflabel['_source']
if preflabel['_source']['language'].split('-')[0] == lang.split('-')[0]:
ret = preflabel['_source']
if preflabel['_source']['language'] == settings.LANGUAGE_CODE and ret == None:
ret = preflabel['_source']
return default if ret == None else ret
def get_related_resources(resourceid, lang, limit=1000, start=0, allowedtypes=[], is_anon=False):
ret = {
'resource_relationships': [],
'related_resources': []
}
se = SearchEngineFactory().create()
query = Query(se, limit=limit, start=start)
query.add_filter(Terms(field='entityid1', terms=resourceid).dsl, operator='or')
query.add_filter(Terms(field='entityid2', terms=resourceid).dsl, operator='or')
resource_relations = query.search(index='resource_relations', doc_type="all")
entityids = set()
for relation in resource_relations['hits']['hits']:
relation['_source']['preflabel'] = get_preflabel_from_valueid(relation['_source']['relationshiptype'], lang)
ret['resource_relationships'].append(relation['_source'])
entityids.add(relation['_source']['entityid1'])
entityids.add(relation['_source']['entityid2'])
if len(entityids) > 0:
entityids.remove(resourceid)
# can't figure why passing allowed types to doc_type param doesn't work,
# so filter is carried out later
related_resources = se.search(index='entity', doc_type='_all', id=list(entityids))
filtered_ids = []
if related_resources:
for resource in related_resources['docs']:
if not resource['_type'] in allowedtypes:
filtered_ids.append(resource['_source']['entityid'])
continue
if is_anon:
# filter out protected resources if user is anonymous
# (this is basically a subset of the get_protected_entityids below
# they should be combined probably)
from search import get_protection_conceptids
protect_id = get_protection_conceptids(settings.PROTECTION_LEVEL_NODE)
conceptids = [d['conceptid'] for d in resource['_source']['domains']]
if protect_id in conceptids:
filtered_ids.append(resource['_source']['entityid'])
continue
ret['related_resources'].append(resource['_source'])
if len(filtered_ids) > 0:
# remove all relationships in ret that match a filtered id (this lc is yuge but I think concise)
filtered_relationships = [rel for rel in ret['resource_relationships'] if not rel['entityid1'] in filtered_ids and not rel['entityid2'] in filtered_ids]
# update ret values
ret['resource_relationships'] = filtered_relationships
ret['total'] = len(ret['resource_relationships'])
return ret
def get_related_resources(resourceid, lang='en-US', limit=1000, start=0):
ret = {'resource_relationships': [], 'related_resources': []}
se = SearchEngineFactory().create()
query = Query(se, limit=limit, start=start)
query.add_filter(Terms(field='entityid1', terms=resourceid).dsl,
operator='or')
resource_relations = query.search(index='resource_relations',
doc_type='all')
ret['total'] = resource_relations['hits']['total']
for relation in resource_relations['hits']['hits']:
ret['resource_relationships'].append(relation['_source'])
return ret
def get_related_resources(resourceid, lang='en-US', limit=1000, start=0):
ret = {
'resource_relationships': [],
'related_resources': []
}
se = SearchEngineFactory().create()
query = Query(se, limit=limit, start=start)
query.add_filter(Terms(field='entityid1', terms=resourceid).dsl, operator='or')
resource_relations = query.search(index='resource_relations', doc_type='all')
ret['total'] = resource_relations['hits']['total']
for relation in resource_relations['hits']['hits']:
ret['resource_relationships'].append(relation['_source'])
return ret
def get_preflabel_from_conceptid(conceptid, lang):
ret = None
default = {"category": "", "conceptid": "", "language": "", "value": "", "type": "", "id": ""}
se = SearchEngineFactory().create()
query = Query(se)
terms = Terms(field="conceptid", terms=[conceptid])
match = Match(field="type", query="preflabel", type="phrase")
query.add_filter(terms)
query.add_query(match)
preflabels = query.search(index="concept_labels")["hits"]["hits"]
for preflabel in preflabels:
default = preflabel["_source"]
# get the label in the preferred language, otherwise get the label in the default language
if preflabel["_source"]["language"] == lang:
return preflabel["_source"]
if preflabel["_source"]["language"].split("-")[0] == lang.split("-")[0]:
ret = preflabel["_source"]
if preflabel["_source"]["language"] == settings.LANGUAGE_CODE and ret == None:
ret = preflabel["_source"]
return default if ret == None else ret
def get_related_resource_ids(resourceids, lang, limit=1000, start=0):
se = SearchEngineFactory().create()
query = Query(se, limit=limit, start=start)
query.add_filter(Terms(field='entityid1', terms=resourceids).dsl, operator='or')
query.add_filter(Terms(field='entityid2', terms=resourceids).dsl, operator='or')
resource_relations = query.search( index='resource_relations', doc_type='all')
entityids = set()
for relation in resource_relations['hits']['hits']:
# add the other halves add the relations which are not in the original list of ids
from_is_original_result = relation['_source']['entityid1'] in resourceids
to_is_original_result = relation['_source']['entityid2'] in resourceids
if from_is_original_result:
entityids.add(relation['_source']['entityid2'])
if to_is_original_result:
entityids.add(relation['_source']['entityid1'])
return entityids
def get_related_resource_ids(resourceids, lang, limit=1000, start=0):
se = SearchEngineFactory().create()
query = Query(se, limit=limit, start=start)
query.add_filter(Terms(field='entityid1', terms=resourceids).dsl,
operator='or')
query.add_filter(Terms(field='entityid2', terms=resourceids).dsl,
operator='or')
resource_relations = query.search(index='resource_relations',
doc_type='all')
entityids = set()
for relation in resource_relations['hits']['hits']:
# add the other halves add the relations which are not in the original list of ids
from_is_original_result = relation['_source'][
'entityid1'] in resourceids
to_is_original_result = relation['_source']['entityid2'] in resourceids
if from_is_original_result:
entityids.add(relation['_source']['entityid2'])
if to_is_original_result:
entityids.add(relation['_source']['entityid1'])
return entityids
def build_search_results_dsl(request):
# Results are sorted ascendingly by the value of SITE_ID.E42, which is displayed as primary name of Heritage Resources.
# Must go back to this method once new Automatic Resource ID has been fully developed (AZ 10/08/16) Update 06/09/16: EAMENA_ID.E42 now used as sorting criterion.
sorting = {
"child_entities.label": {
"order" : "asc",
"nested_path": "child_entities",
"nested_filter": {
"term": {"child_entities.entitytypeid" : "EAMENA_ID.E42"}
}
}
}
term_filter = request.GET.get('termFilter', '')
spatial_filter = JSONDeserializer().deserialize(request.GET.get('spatialFilter', None))
export = request.GET.get('export', None)
page = 1 if request.GET.get('page') == '' else int(request.GET.get('page', 1))
temporal_filter = JSONDeserializer().deserialize(request.GET.get('temporalFilter', None))
boolean_search = request.GET.get('booleanSearch', '')
filter_and_or = JSONDeserializer().deserialize(request.GET.get('termFilterAndOr', ''))
filter_grouping = JSONDeserializer().deserialize(request.GET.get('termFilterGroup', ''))
filter_combine_flags = JSONDeserializer().deserialize(request.GET.get('termFilterCombineWithPrev', ''))
#Ignore first entry as it is a dummy
filter_combine_flags = filter_combine_flags[1:]
# filter_combine_flags = [False, True, False, False, False]
# filter_groups = JSONDeserializer().deserialize(request.GET.get('termFilterGroups', ''))
# Not here yet, so put in some bogus data
# filter_groups = [
# 'NAME.E41',
# 'NAME.E41',
# 'DISTURBANCE_STATE.E3',
# 'THREAT_STATE.E3'
# ]
se = SearchEngineFactory().create()
if export != None:
limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE
else:
limit = settings.SEARCH_ITEMS_PER_PAGE
query = Query(se, start=limit*int(page-1), limit=limit)
boolquery = Bool()
boolfilter = Bool()
is_empty_temporal_filter = True
# store each search term in an initially. These will be combined based on the global and/or and the optional groupings
terms_queries = [];
# logging.warning("-------QUERY-------")
if term_filter != '' or not is_empty_temporal_filter:
for index, select_box in enumerate(JSONDeserializer().deserialize(term_filter)):
selectbox_boolfilter = Bool()
groupid = filter_grouping[index]
if not groupid == 'No group':
# build a nested query against the nested_entities
# build a nested query for each resource type
for resourcetype in settings.RESOURCE_TYPE_CONFIGS().keys():
# trace the path from each term to the group root
term_paths = []
for term in select_box:
# trace path from group root to this term
if term['type'] == 'concept':
# get all the parent concepts for this value i.e. the field
concept_relations = models.ConceptRelations.objects.filter(conceptidto=term['value'], relationtype="member")
for relation in concept_relations:
term_parent_concept = models.Concepts.objects.get(conceptid=relation.conceptidfrom)
# get the steps from the root to that concept
if term_parent_concept.nodetype.nodetype == "Collection":
term_schema = Entity.get_mapping_schema_to(term_parent_concept.legacyoid)
elif term_parent_concept.nodetype.nodetype == 'Concept':
# need to get at the parent until we reach the root collection. concepts are arranged hierarchically
parent_relations_to = models.ConceptRelations.objects.filter(conceptidto=term_parent_concept.conceptid, relationtype='member')
grandparent = models.Concepts.objects.filter(conceptid=parent_relations_to[0].conceptidfrom)
term_schema = Entity.get_mapping_schema_to(grandparent[0].legacyoid)
#this path begins at the root, and ends up at the node in question
if resourcetype in term_schema:
term_path = term_schema[resourcetype]['steps']
term_paths.append({
'term': term,
'path': term_path
})
break
elif term['type'] == 'term':
concept = models.Concepts.objects.get(conceptid=term['context'])
term_schema = Entity.get_mapping_schema_to(concept.legacyoid)
if resourcetype in term_schema:
term_path = term_schema[resourcetype]['steps']
term_paths.append({
'term': term,
'path': term_path
})
elif term['type'] == 'string':
term_schema = Entity.get_mapping_schema_to(groupid)
if resourcetype in term_schema:
term_path = term_schema[resourcetype]['steps']
term_paths.append({
'term': term,
'path': term_path
})
if 'year_min_max' in temporal_filter[index] and len(temporal_filter[index]['year_min_max']) == 2:
start_date = date(temporal_filter[index]['year_min_max'][0], 1, 1)
end_date = date(temporal_filter[index]['year_min_max'][1], 12, 31)
if start_date:
start_date = start_date.isoformat()
if end_date:
end_date = end_date.isoformat()
if 'inverted' not in temporal_filter[index]:
inverted_temporal_filter = False
else:
if temporal_filter[index]['inverted']:
inverted_temporal_filter = True
else:
inverted_temporal_filter = False
term_paths.append({
'term': {
'date_operator': '3',
'start_date': start_date,
'end_date': end_date,
'type': 'date',
'inverted': inverted_temporal_filter
},
'path': term_path
})
if 'filters' in temporal_filter[index]:
term_schema = Entity.get_mapping_schema_to(groupid)
if resourcetype in term_schema:
term_path = term_schema[resourcetype]['steps']
for temporal_filter_item in temporal_filter[index]['filters']:
date_type = ''
searchdate = ''
date_operator = ''
for node in temporal_filter_item['nodes']:
if node['entitytypeid'] == 'DATE_COMPARISON_OPERATOR.E55':
date_operator = node['value']
elif node['entitytypeid'] == 'date':
searchdate = node['value']
else:
date_type = node['value']
date_value = datetime.strptime(searchdate, '%Y-%m-%d').isoformat()
if 'inverted' not in temporal_filter[index]:
inverted_temporal_filter = False
else:
if temporal_filter[index]['inverted']:
inverted_temporal_filter = True
else:
inverted_temporal_filter = False
term_paths.append({
'term': {
'date_operator': date_operator,
'date_value': date_value,
'type': 'date',
'inverted': inverted_temporal_filter
},
'path': term_path
})
# combine the traced path to build a nested query
group_query = nested_query_from_pathed_values(term_paths, 'nested_entity.child_entities')
# add nested query to overall query
selectbox_boolfilter.should(group_query)
# logging.warning("BOX QUERY - %s", JSONSerializer().serialize(selectbox_boolfilter, indent=2))
else:
for term in select_box:
if term['type'] == 'term':
entitytype = models.EntityTypes.objects.get(conceptid_id=term['context'])
boolfilter_nested = Bool()
boolfilter_nested.must(Terms(field='child_entities.entitytypeid', terms=[entitytype.pk]))
boolfilter_nested.must(Match(field='child_entities.value', query=term['value'], type='phrase'))
nested = Nested(path='child_entities', query=boolfilter_nested)
if filter_and_or[index] == 'or':
if not term['inverted']:
selectbox_boolfilter.should(nested)
else:
if term['inverted']:
selectbox_boolfilter.must_not(nested)
else:
selectbox_boolfilter.must(nested)
elif term['type'] == 'concept':
concept_ids = _get_child_concepts(term['value'])
terms = Terms(field='domains.conceptid', terms=concept_ids)
nested = Nested(path='domains', query=terms)
if filter_and_or[index] == 'or':
if not term['inverted']:
selectbox_boolfilter.should(nested)
else:
if term['inverted']:
selectbox_boolfilter.must_not(nested)
else:
selectbox_boolfilter.must(nested)
elif term['type'] == 'string':
boolquery2 = Bool() #This bool contains the subset of nested string queries on both domains and child_entities paths
boolfilter_folded = Bool() #This bool searches by string in child_entities, where free text strings get indexed
boolfilter_folded2 = Bool() #This bool searches by string in the domains path,where controlled vocabulary concepts get indexed
boolfilter_folded.should(Match(field='child_entities.value', query=term['value'], type='phrase_prefix', fuzziness='AUTO', operator='and'))
boolfilter_folded.should(Match(field='child_entities.value.folded', query=term['value'], type='phrase_prefix', fuzziness='AUTO', operator='and'))
boolfilter_folded.should(Match(field='child_entities.value.folded', query=term['value'], fuzziness='AUTO', operator='and'))
nested = Nested(path='child_entities', query=boolfilter_folded)
boolfilter_folded2.should(Match(field='domains.label', query=term['value'], type='phrase_prefix', fuzziness='AUTO', operator='and'))
boolfilter_folded2.should(Match(field='domains.label.folded', query=term['value'], type='phrase_prefix', fuzziness='AUTO', operator='and'))
boolfilter_folded2.should(Match(field='domains.label.folded', query=term['value'], fuzziness='AUTO', operator='and'))
nested2 = Nested(path='domains', query=boolfilter_folded2)
boolquery2.should(nested)
boolquery2.should(nested2)
if filter_and_or[index] == 'or':
if not term['inverted']:
# use boolfilter here instead of boolquery because boolquery
# can't be combined with other boolfilters using boolean OR
selectbox_boolfilter.should(boolquery2)
else:
if term['inverted']:
selectbox_boolfilter.must_not(boolquery2)
else:
selectbox_boolfilter.must(boolquery2)
if 'year_min_max' in temporal_filter[index] and len(temporal_filter[index]['year_min_max']) == 2:
start_date = date(temporal_filter[index]['year_min_max'][0], 1, 1)
end_date = date(temporal_filter[index]['year_min_max'][1], 12, 31)
if start_date:
start_date = start_date.isoformat()
if end_date:
end_date = end_date.isoformat()
range = Range(field='dates.value', gte=start_date, lte=end_date)
nested = Nested(path='dates', query=range)
if 'inverted' not in temporal_filter[index]:
temporal_filter[index]['inverted'] = False
if temporal_filter[index]['inverted']:
selectbox_boolfilter.must_not(nested)
else:
selectbox_boolfilter.must(nested)
if 'filters' in temporal_filter[index]:
for temporal_filter_item in temporal_filter[index]['filters']:
date_type = ''
searchdate = ''
date_operator = ''
for node in temporal_filter_item['nodes']:
if node['entitytypeid'] == 'DATE_COMPARISON_OPERATOR.E55':
date_operator = node['value']
elif node['entitytypeid'] == 'date':
searchdate = node['value']
else:
date_type = node['value']
date_value = datetime.strptime(searchdate, '%Y-%m-%d').isoformat()
if date_operator == '1': # equals query
range = Range(field='dates.value', gte=date_value, lte=date_value)
elif date_operator == '0': # greater than query
range = Range(field='dates.value', lt=date_value)
elif date_operator == '2': # less than query
range = Range(field='dates.value', gt=date_value)
nested = Nested(path='dates', query=range)
if 'inverted' not in temporal_filter[index]:
temporal_filter[index]['inverted'] = False
if temporal_filter[index]['inverted']:
selectbox_boolfilter.must_not(nested)
else:
selectbox_boolfilter.must(nested)
terms_queries.append(selectbox_boolfilter)
# if not selectbox_boolfilter.empty:
# if boolean_search == 'or':
# boolfilter.should(selectbox_boolfilter)
# else:
# boolfilter.must(selectbox_boolfilter)
# We now have individual query terms for each of the search components. Combine into one group now
# Start by building a an array of groups which will be combined according to the global And/Or
# Queries within one of these groups will be combined by the complement of the global And/Or
# We may end up with [ [A,B], [C], [D,E] ], which would translate to either:
# (A || B) && C && (D || E)
# or
# (A && B) || C || (D && E)
# for global AND or OR respectively
# logging.warning("TERMS QUERIES %s", terms_queries)
bool_components = [];
for i, term_query in enumerate(terms_queries):
if i is 0:
bool_components.append([term_query])
else:
should_group_with_previous = filter_combine_flags[i-1]
if should_group_with_previous:
bool_components[-1].append(term_query)
else:
bool_components.append([term_query])
# logging.warning("BOOL COMPONENTS %s", bool_components)
# Now build the ES queries
for bool_component in bool_components:
if len(bool_component) is 1:
# just combine this on its own
q = bool_component[0]
else:
q = Bool()
for sub_component in bool_component:
if boolean_search == 'or':
#apply the OPPOSITE of the global boolean operator
q.must(sub_component)
else:
q.should(sub_component)
# combine to the overall query according to the global boolean operator
if boolean_search == 'or':
boolfilter.should(q)
else:
boolfilter.must(q)
if 'geometry' in spatial_filter and 'type' in spatial_filter['geometry'] and spatial_filter['geometry']['type'] != '':
geojson = spatial_filter['geometry']
if geojson['type'] == 'bbox':
coordinates = [[geojson['coordinates'][0],geojson['coordinates'][3]], [geojson['coordinates'][2],geojson['coordinates'][1]]]
geoshape = GeoShape(field='geometries.value', type='envelope', coordinates=coordinates )
nested = Nested(path='geometries', query=geoshape)
else:
buffer = spatial_filter['buffer']
geojson = JSONDeserializer().deserialize(_buffer(geojson,buffer['width'],buffer['unit']).json)
geoshape = GeoShape(field='geometries.value', type=geojson['type'], coordinates=geojson['coordinates'] )
nested = Nested(path='geometries', query=geoshape)
if 'inverted' not in spatial_filter:
spatial_filter['inverted'] = False
if spatial_filter['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
if not boolquery.empty:
query.add_query(boolquery)
if not boolfilter.empty:
query.add_filter(boolfilter)
# Sorting criterion added to query (AZ 10/08/16)
query.dsl.update({'sort': sorting})
# logging.warning("-=-==-=-===-=--=-==-=-===-=- query: -=-==-=-===-=--=-==-=-===-=-> %s", query)
return query
def build_search_results_dsl(request):
term_filter = request.GET.get('termFilter', '')
spatial_filter = JSONDeserializer().deserialize(
request.GET.get('mapFilter', '{}'))
export = request.GET.get('export', None)
page = 1 if request.GET.get('page') == '' else int(
request.GET.get('page', 1))
temporal_filter = JSONDeserializer().deserialize(
request.GET.get('temporalFilter', '{}'))
se = SearchEngineFactory().create()
if export != None:
limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE
else:
limit = settings.SEARCH_ITEMS_PER_PAGE
query = Query(se, start=limit * int(page - 1), limit=limit)
boolquery = Bool()
boolfilter = Bool()
if term_filter != '':
for term in JSONDeserializer().deserialize(term_filter):
if term['type'] == 'term':
term_filter = Bool()
term_filter.must(
Match(field='strings', query=term['value'], type='phrase'))
if term['inverted']:
boolfilter.must_not(term_filter)
else:
boolfilter.must(term_filter)
elif term['type'] == 'concept':
concept_ids = _get_child_concepts(term['value'])
conceptid_filter = Terms(field='domains.conceptid',
terms=concept_ids)
if term['inverted']:
boolfilter.must_not(conceptid_filter)
else:
boolfilter.must(conceptid_filter)
elif term['type'] == 'string':
string_filter = Bool()
string_filter.should(
Match(field='strings',
query=term['value'],
type='phrase_prefix'))
string_filter.should(
Match(field='strings.folded',
query=term['value'],
type='phrase_prefix'))
if term['inverted']:
boolfilter.must_not(string_filter)
else:
boolfilter.must(string_filter)
if 'features' in spatial_filter:
if len(spatial_filter['features']) > 0:
feature_geom = spatial_filter['features'][0]['geometry']
feature_properties = spatial_filter['features'][0]['properties']
buffer = {'width': 0, 'unit': 'ft'}
if 'buffer' in feature_properties:
buffer = feature_properties['buffer']
feature_geom = JSONDeserializer().deserialize(
_buffer(feature_geom, buffer['width'], buffer['unit']).json)
geoshape = GeoShape(field='geometries.features.geometry',
type=feature_geom['type'],
coordinates=feature_geom['coordinates'])
invert_spatial_search = False
if 'inverted' in feature_properties:
invert_spatial_search = feature_properties['inverted']
if invert_spatial_search == True:
boolfilter.must_not(geoshape)
else:
boolfilter.must(geoshape)
if 'fromDate' in temporal_filter and 'toDate' in temporal_filter:
start_date = None
end_date = None
try:
start_date = parser.parse(temporal_filter['fromDate'])
start_date = start_date.isoformat()
except:
pass
try:
end_date = parser.parse(temporal_filter['toDate'])
end_date = end_date.isoformat()
except:
pass
if 'dateNodeId' in temporal_filter and temporal_filter[
'dateNodeId'] != '':
range = Range(field='tiles.data.%s' %
(temporal_filter['dateNodeId']),
gte=start_date,
lte=end_date)
time_query_dsl = Nested(path='tiles', query=range)
else:
time_query_dsl = Range(field='dates', gte=start_date, lte=end_date)
if 'inverted' not in temporal_filter:
temporal_filter['inverted'] = False
if temporal_filter['inverted']:
boolfilter.must_not(time_query_dsl)
else:
boolfilter.must(time_query_dsl)
if not boolquery.empty:
query.add_query(boolquery)
if not boolfilter.empty:
query.add_filter(boolfilter)
return query
def build_search_results_dsl(request):
term_filter = request.GET.get('termFilter', '')
spatial_filter = JSONDeserializer().deserialize(request.GET.get('spatialFilter', None))
export = request.GET.get('export', None)
page = 1 if request.GET.get('page') == '' else int(request.GET.get('page', 1))
temporal_filter = JSONDeserializer().deserialize(request.GET.get('temporalFilter', None))
se = SearchEngineFactory().create()
if export != None:
limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE
else:
limit = settings.SEARCH_ITEMS_PER_PAGE
query = Query(se, start=limit*int(page-1), limit=limit)
boolquery = Bool()
boolfilter = Bool()
if term_filter != '':
for term in JSONDeserializer().deserialize(term_filter):
if term['type'] == 'term':
entitytype = models.EntityTypes.objects.get(conceptid_id=term['context'])
boolfilter_nested = Bool()
boolfilter_nested.must(Terms(field='child_entities.entitytypeid', terms=[entitytype.pk]))
boolfilter_nested.must(Match(field='child_entities.value', query=term['value'], type='phrase'))
nested = Nested(path='child_entities', query=boolfilter_nested)
if term['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
elif term['type'] == 'concept':
concept_ids = _get_child_concepts(term['value'])
terms = Terms(field='domains.conceptid', terms=concept_ids)
nested = Nested(path='domains', query=terms)
if term['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
elif term['type'] == 'string':
boolfilter_folded = Bool()
boolfilter_folded.should(Match(field='child_entities.value', query=term['value'], type='phrase_prefix'))
boolfilter_folded.should(Match(field='child_entities.value.folded', query=term['value'], type='phrase_prefix'))
nested = Nested(path='child_entities', query=boolfilter_folded)
if term['inverted']:
boolquery.must_not(nested)
else:
boolquery.must(nested)
if 'geometry' in spatial_filter and 'type' in spatial_filter['geometry'] and spatial_filter['geometry']['type'] != '':
geojson = spatial_filter['geometry']
if geojson['type'] == 'bbox':
coordinates = [[geojson['coordinates'][0],geojson['coordinates'][3]], [geojson['coordinates'][2],geojson['coordinates'][1]]]
geoshape = GeoShape(field='geometries.value', type='envelope', coordinates=coordinates )
nested = Nested(path='geometries', query=geoshape)
else:
buffer = spatial_filter['buffer']
geojson = JSONDeserializer().deserialize(_buffer(geojson,buffer['width'],buffer['unit']).json)
geoshape = GeoShape(field='geometries.value', type=geojson['type'], coordinates=geojson['coordinates'] )
nested = Nested(path='geometries', query=geoshape)
if 'inverted' not in spatial_filter:
spatial_filter['inverted'] = False
if spatial_filter['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
if 'year_min_max' in temporal_filter and len(temporal_filter['year_min_max']) == 2:
start_date = date(temporal_filter['year_min_max'][0], 1, 1)
end_date = date(temporal_filter['year_min_max'][1], 12, 31)
if start_date:
start_date = start_date.isoformat()
if end_date:
end_date = end_date.isoformat()
range = Range(field='dates.value', gte=start_date, lte=end_date)
nested = Nested(path='dates', query=range)
if 'inverted' not in temporal_filter:
temporal_filter['inverted'] = False
if temporal_filter['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
if not boolquery.empty:
query.add_query(boolquery)
if not boolfilter.empty:
query.add_filter(boolfilter)
return query
def build_search_results_dsl(request):
# Results are sorted ascendingly by the value of SITE_ID.E42, which is displayed as primary name of Heritage Resources.
# Must go back to this method once new Automatic Resource ID has been fully developed (AZ 10/08/16) Update 06/09/16: EAMENA_ID.E42 now used as sorting criterion.
sorting = {
"child_entities.label": {
"order" : "asc",
"nested_path": "child_entities",
"nested_filter": {
"term": {"child_entities.entitytypeid" : "EAMENA_ID.E42"}
}
}
}
term_filter = request.GET.get('termFilter', '')
spatial_filter = JSONDeserializer().deserialize(request.GET.get('spatialFilter', None))
export = request.GET.get('export', None)
page = 1 if request.GET.get('page') == '' else int(request.GET.get('page', 1))
temporal_filter = JSONDeserializer().deserialize(request.GET.get('temporalFilter', None))
boolean_search = request.GET.get('booleanSearch', '')
filter_and_or = JSONDeserializer().deserialize(request.GET.get('termFilterAndOr', ''))
filter_grouping = JSONDeserializer().deserialize(request.GET.get('termFilterGroup', ''))
filter_combine_flags = JSONDeserializer().deserialize(request.GET.get('termFilterCombineWithPrev', ''))
#Ignore first entry as it is a dummy
filter_combine_flags = filter_combine_flags[1:]
# filter_combine_flags = [False, True, False, False, False]
# filter_groups = JSONDeserializer().deserialize(request.GET.get('termFilterGroups', ''))
# Not here yet, so put in some bogus data
# filter_groups = [
# 'NAME.E41',
# 'NAME.E41',
# 'DISTURBANCE_STATE.E3',
# 'THREAT_STATE.E3'
# ]
se = SearchEngineFactory().create()
if export != None:
limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE
else:
limit = settings.SEARCH_ITEMS_PER_PAGE
query = Query(se, start=limit*int(page-1), limit=limit)
boolquery = Bool()
boolfilter = Bool()
is_empty_temporal_filter = True
# store each search term in an initially. These will be combined based on the global and/or and the optional groupings
terms_queries = [];
# logging.warning("-------QUERY-------")
if term_filter != '' or not is_empty_temporal_filter:
for index, select_box in enumerate(JSONDeserializer().deserialize(term_filter)):
selectbox_boolfilter = Bool()
groupid = filter_grouping[index]
if not groupid == 'No group':
# build a nested query against the nested_entities
# trace the path from each term to the group root
term_paths = []
for term in select_box:
# trace path from group root to this term
if term['type'] == 'concept':
# get the parent concept for this value i.e. the field
term_parent_concept = Concept.get_parent_concept(term['value'])
# get the steps from the root to that concept
if term_parent_concept.nodetype.nodetype == "Collection":
term_schema = Entity.get_mapping_schema_to(term_parent_concept.legacyoid)
elif term_parent_concept.nodetype.nodetype == 'Concept':
# need to get at the parent until we reach the root collection. concepts are arranged hierarchically
parent_relations_to = models.ConceptRelations.objects.filter(conceptidto=term_parent_concept.conceptid, relationtype='member')
grandparent = models.Concepts.objects.filter(conceptid=parent_relations_to[0].conceptidfrom)
term_schema = Entity.get_mapping_schema_to(grandparent[0].legacyoid)
#this path begins at the root, and ends up at the node in question
term_path = term_schema['HERITAGE_RESOURCE_GROUP.E27']['steps']
term_paths.append({
'term': term,
'path': term_path
})
elif term['type'] == 'term':
concept = models.Concepts.objects.get(conceptid=term['context'])
term_schema = Entity.get_mapping_schema_to(concept.legacyoid)
term_path = term_schema['HERITAGE_RESOURCE_GROUP.E27']['steps']
term_paths.append({
'term': term,
'path': term_path
})
elif term['type'] == 'string':
term_schema = Entity.get_mapping_schema_to(groupid)
term_path = term_schema['HERITAGE_RESOURCE_GROUP.E27']['steps']
term_paths.append({
'term': term,
'path': term_path
})
if 'year_min_max' in temporal_filter[index] and len(temporal_filter[index]['year_min_max']) == 2:
start_date = date(temporal_filter[index]['year_min_max'][0], 1, 1)
end_date = date(temporal_filter[index]['year_min_max'][1], 12, 31)
if start_date:
start_date = start_date.isoformat()
if end_date:
end_date = end_date.isoformat()
if 'inverted' not in temporal_filter[index]:
inverted_temporal_filter = False
else:
if temporal_filter[index]['inverted']:
inverted_temporal_filter = True
else:
inverted_temporal_filter = False
term_paths.append({
'term': {
'date_operator': '3',
'start_date': start_date,
'end_date': end_date,
'type': 'date',
'inverted': inverted_temporal_filter
},
'path': term_path
})
if 'filters' in temporal_filter[index]:
term_schema = Entity.get_mapping_schema_to(groupid)
term_path = term_schema['HERITAGE_RESOURCE_GROUP.E27']['steps']
for temporal_filter_item in temporal_filter[index]['filters']:
date_type = ''
searchdate = ''
date_operator = ''
for node in temporal_filter_item['nodes']:
if node['entitytypeid'] == 'DATE_COMPARISON_OPERATOR.E55':
date_operator = node['value']
elif node['entitytypeid'] == 'date':
searchdate = node['value']
else:
date_type = node['value']
date_value = datetime.strptime(searchdate, '%Y-%m-%d').isoformat()
if 'inverted' not in temporal_filter[index]:
inverted_temporal_filter = False
else:
if temporal_filter[index]['inverted']:
inverted_temporal_filter = True
else:
inverted_temporal_filter = False
term_paths.append({
'term': {
'date_operator': date_operator,
'date_value': date_value,
'type': 'date',
'inverted': inverted_temporal_filter
},
'path': term_path
})
# combine the traced path to build a nested query
group_query = nested_query_from_pathed_values(term_paths, 'nested_entity.child_entities')
# add nested query to overall query
selectbox_boolfilter.must(group_query)
# logging.warning("BOX QUERY - %s", JSONSerializer().serialize(selectbox_boolfilter, indent=2))
else:
for term in select_box:
if term['type'] == 'term':
entitytype = models.EntityTypes.objects.get(conceptid_id=term['context'])
boolfilter_nested = Bool()
boolfilter_nested.must(Terms(field='child_entities.entitytypeid', terms=[entitytype.pk]))
boolfilter_nested.must(Match(field='child_entities.value', query=term['value'], type='phrase'))
nested = Nested(path='child_entities', query=boolfilter_nested)
if filter_and_or[index] == 'or':
if not term['inverted']:
selectbox_boolfilter.should(nested)
else:
if term['inverted']:
selectbox_boolfilter.must_not(nested)
else:
selectbox_boolfilter.must(nested)
elif term['type'] == 'concept':
concept_ids = _get_child_concepts(term['value'])
terms = Terms(field='domains.conceptid', terms=concept_ids)
nested = Nested(path='domains', query=terms)
if filter_and_or[index] == 'or':
if not term['inverted']:
selectbox_boolfilter.should(nested)
else:
if term['inverted']:
selectbox_boolfilter.must_not(nested)
else:
selectbox_boolfilter.must(nested)
elif term['type'] == 'string':
boolquery2 = Bool() #This bool contains the subset of nested string queries on both domains and child_entities paths
boolfilter_folded = Bool() #This bool searches by string in child_entities, where free text strings get indexed
boolfilter_folded2 = Bool() #This bool searches by string in the domains path,where controlled vocabulary concepts get indexed
boolfilter_folded.should(Match(field='child_entities.value', query=term['value'], type='phrase_prefix', fuzziness='AUTO', operator='and'))
boolfilter_folded.should(Match(field='child_entities.value.folded', query=term['value'], type='phrase_prefix', fuzziness='AUTO', operator='and'))
boolfilter_folded.should(Match(field='child_entities.value.folded', query=term['value'], fuzziness='AUTO', operator='and'))
nested = Nested(path='child_entities', query=boolfilter_folded)
boolfilter_folded2.should(Match(field='domains.label', query=term['value'], type='phrase_prefix', fuzziness='AUTO', operator='and'))
boolfilter_folded2.should(Match(field='domains.label.folded', query=term['value'], type='phrase_prefix', fuzziness='AUTO', operator='and'))
boolfilter_folded2.should(Match(field='domains.label.folded', query=term['value'], fuzziness='AUTO', operator='and'))
nested2 = Nested(path='domains', query=boolfilter_folded2)
boolquery2.should(nested)
boolquery2.should(nested2)
if filter_and_or[index] == 'or':
if not term['inverted']:
# use boolfilter here instead of boolquery because boolquery
# can't be combined with other boolfilters using boolean OR
selectbox_boolfilter.should(boolquery2)
else:
if term['inverted']:
selectbox_boolfilter.must_not(boolquery2)
else:
selectbox_boolfilter.must(boolquery2)
if 'year_min_max' in temporal_filter[index] and len(temporal_filter[index]['year_min_max']) == 2:
start_date = date(temporal_filter[index]['year_min_max'][0], 1, 1)
end_date = date(temporal_filter[index]['year_min_max'][1], 12, 31)
if start_date:
start_date = start_date.isoformat()
if end_date:
end_date = end_date.isoformat()
range = Range(field='dates.value', gte=start_date, lte=end_date)
nested = Nested(path='dates', query=range)
if 'inverted' not in temporal_filter[index]:
temporal_filter[index]['inverted'] = False
if temporal_filter[index]['inverted']:
selectbox_boolfilter.must_not(nested)
else:
selectbox_boolfilter.must(nested)
if 'filters' in temporal_filter[index]:
for temporal_filter_item in temporal_filter[index]['filters']:
date_type = ''
searchdate = ''
date_operator = ''
for node in temporal_filter_item['nodes']:
if node['entitytypeid'] == 'DATE_COMPARISON_OPERATOR.E55':
date_operator = node['value']
elif node['entitytypeid'] == 'date':
searchdate = node['value']
else:
date_type = node['value']
date_value = datetime.strptime(searchdate, '%Y-%m-%d').isoformat()
if date_operator == '1': # equals query
range = Range(field='dates.value', gte=date_value, lte=date_value)
elif date_operator == '0': # greater than query
range = Range(field='dates.value', lt=date_value)
elif date_operator == '2': # less than query
range = Range(field='dates.value', gt=date_value)
nested = Nested(path='dates', query=range)
if 'inverted' not in temporal_filter[index]:
temporal_filter[index]['inverted'] = False
if temporal_filter[index]['inverted']:
selectbox_boolfilter.must_not(nested)
else:
selectbox_boolfilter.must(nested)
terms_queries.append(selectbox_boolfilter)
# if not selectbox_boolfilter.empty:
# if boolean_search == 'or':
# boolfilter.should(selectbox_boolfilter)
# else:
# boolfilter.must(selectbox_boolfilter)
# We now have individual query terms for each of the search components. Combine into one group now
# Start by building a an array of groups which will be combined according to the global And/Or
# Queries within one of these groups will be combined by the complement of the global And/Or
# We may end up with [ [A,B], [C], [D,E] ], which would translate to either:
# (A || B) && C && (D || E)
# or
# (A && B) || C || (D && E)
# for global AND or OR respectively
# logging.warning("TERMS QUERIES %s", terms_queries)
bool_components = [];
for i, term_query in enumerate(terms_queries):
if i is 0:
bool_components.append([term_query])
else:
should_group_with_previous = filter_combine_flags[i-1]
if should_group_with_previous:
bool_components[-1].append(term_query)
else:
bool_components.append([term_query])
# logging.warning("BOOL COMPONENTS %s", bool_components)
# Now build the ES queries
for bool_component in bool_components:
if len(bool_component) is 1:
# just combine this on its own
q = bool_component[0]
else:
q = Bool()
for sub_component in bool_component:
if boolean_search == 'or':
#apply the OPPOSITE of the global boolean operator
q.must(sub_component)
else:
q.should(sub_component)
# combine to the overall query according to the global boolean operator
if boolean_search == 'or':
boolfilter.should(q)
else:
boolfilter.must(q)
if 'geometry' in spatial_filter and 'type' in spatial_filter['geometry'] and spatial_filter['geometry']['type'] != '':
geojson = spatial_filter['geometry']
if geojson['type'] == 'bbox':
coordinates = [[geojson['coordinates'][0],geojson['coordinates'][3]], [geojson['coordinates'][2],geojson['coordinates'][1]]]
geoshape = GeoShape(field='geometries.value', type='envelope', coordinates=coordinates )
nested = Nested(path='geometries', query=geoshape)
else:
buffer = spatial_filter['buffer']
geojson = JSONDeserializer().deserialize(_buffer(geojson,buffer['width'],buffer['unit']).json)
geoshape = GeoShape(field='geometries.value', type=geojson['type'], coordinates=geojson['coordinates'] )
nested = Nested(path='geometries', query=geoshape)
if 'inverted' not in spatial_filter:
spatial_filter['inverted'] = False
if spatial_filter['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
if not boolquery.empty:
query.add_query(boolquery)
if not boolfilter.empty:
query.add_filter(boolfilter)
# Sorting criterion added to query (AZ 10/08/16)
query.dsl.update({'sort': sorting})
# logging.warning("-=-==-=-===-=--=-==-=-===-=- query: -=-==-=-===-=--=-==-=-===-=-> %s", query)
return query
def build_search_results_dsl(request):
term_filter = request.GET.get('termFilter', '')
spatial_filter = JSONDeserializer().deserialize(request.GET.get('spatialFilter', None))
export = request.GET.get('export', None)
page = 1 if request.GET.get('page') == '' else int(request.GET.get('page', 1))
temporal_filter = JSONDeserializer().deserialize(request.GET.get('temporalFilter', None))
se = SearchEngineFactory().create()
if export != None:
limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE
else:
limit = settings.SEARCH_ITEMS_PER_PAGE
query = Query(se, start=limit*int(page-1), limit=limit)
boolquery = Bool()
boolfilter = Bool()
if term_filter != '':
for term in JSONDeserializer().deserialize(term_filter):
if term['type'] == 'term':
entitytype = models.EntityTypes.objects.get(conceptid_id=term['context'])
boolfilter_nested = Bool()
boolfilter_nested.must(Terms(field='child_entities.entitytypeid', terms=[entitytype.pk]))
boolfilter_nested.must(Match(field='child_entities.value', query=term['value'], type='phrase'))
nested = Nested(path='child_entities', query=boolfilter_nested)
if term['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
elif term['type'] == 'concept':
concept_ids = _get_child_concepts(term['value'])
terms = Terms(field='domains.conceptid', terms=concept_ids)
nested = Nested(path='domains', query=terms)
if term['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
elif term['type'] == 'string':
boolfilter_folded = Bool()
boolfilter_folded.should(Match(field='child_entities.value', query=term['value'], type='phrase_prefix'))
boolfilter_folded.should(Match(field='child_entities.value.folded', query=term['value'], type='phrase_prefix'))
nested = Nested(path='child_entities', query=boolfilter_folded)
if term['inverted']:
boolquery.must_not(nested)
else:
boolquery.must(nested)
if 'geometry' in spatial_filter and 'type' in spatial_filter['geometry'] and spatial_filter['geometry']['type'] != '':
geojson = spatial_filter['geometry']
if geojson['type'] == 'bbox':
coordinates = [[geojson['coordinates'][0],geojson['coordinates'][3]], [geojson['coordinates'][2],geojson['coordinates'][1]]]
geoshape = GeoShape(field='geometries.value', type='envelope', coordinates=coordinates )
nested = Nested(path='geometries', query=geoshape)
else:
buffer = spatial_filter['buffer']
geojson = JSONDeserializer().deserialize(_buffer(geojson,buffer['width'],buffer['unit']).json)
geoshape = GeoShape(field='geometries.value', type=geojson['type'], coordinates=geojson['coordinates'] )
nested = Nested(path='geometries', query=geoshape)
if 'inverted' not in spatial_filter:
spatial_filter['inverted'] = False
if spatial_filter['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
if 'year_min_max' in temporal_filter and len(temporal_filter['year_min_max']) == 2:
start = temporal_filter['year_min_max'][0]*10000
end = temporal_filter['year_min_max'][1]*10000
range = Range(field='extendeddates.value', gte=start, lte=end)
nested = Nested(path='extendeddates', query=range)
if 'inverted' not in temporal_filter:
temporal_filter['inverted'] = False
if temporal_filter['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
if 'filters' in temporal_filter:
time_boolfilter = Bool()
for temporal_filter in temporal_filter['filters']:
date_type = ''
date = ''
date_operator = ''
for node in temporal_filter['nodes']:
if node['entitytypeid'] == 'DATE_COMPARISON_OPERATOR.E55':
date_operator = node['value']
elif node['entitytypeid'] == 'date':
date = node['value']
else:
date_type = node['value']
terms = Terms(field='extendeddategroups.conceptid', terms=date_type)
boolfilter.must(terms)
date_value = date_to_int(date)
if date_operator == '1': # equals query
range = Range(field='extendeddategroups.value', gte=date_value, lte=date_value)
elif date_operator == '0': # greater than query
range = Range(field='extendeddategroups.value', lt=date_value)
elif date_operator == '2': # less than query
range = Range(field='extendeddategroups.value', gt=date_value)
if 'inverted' not in temporal_filter:
temporal_filter['inverted'] = False
if temporal_filter['inverted']:
boolfilter.must_not(range)
else:
boolfilter.must(range)
if not boolquery.empty:
query.add_query(boolquery)
if not boolfilter.empty:
query.add_filter(boolfilter)
return query
def build_base_search_results_dsl(request):
term_filter = request.GET.get('termFilter', '')
spatial_filter = JSONDeserializer().deserialize(request.GET.get('spatialFilter', None))
export = request.GET.get('export', None)
page = 1 if request.GET.get('page') == '' else int(request.GET.get('page', 1))
temporal_filter = JSONDeserializer().deserialize(request.GET.get('temporalFilter', None))
se = SearchEngineFactory().create()
if export != None:
limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE
else:
limit = settings.SEARCH_ITEMS_PER_PAGE
query = Query(se, start=limit*int(page-1), limit=limit)
boolquery = Bool()
boolfilter = Bool()
if term_filter != '':
# Ce uporabnik ni avtenticiran, prikazemo le veljavne (to je verjetno potrebno se dodelati (mogoce da vidijo le svoje???)!!!)
if (request.user.username == 'anonymous'):
auto_filter = []
for item in JSONDeserializer().deserialize(term_filter):
auto_filter.append(item)
# Poiscimo concept id in context za Published status
AUTO_TERM_FILTER = get_auto_filter(request)
auto_filter.append(AUTO_TERM_FILTER)
term_filter = JSONSerializer().serialize(auto_filter)
print 'term_filter'
if term_filter != '':
for term in JSONDeserializer().deserialize(term_filter):
print term
if term['type'] == 'term':
entitytype = models.EntityTypes.objects.get(conceptid_id=term['context'])
boolfilter_nested = Bool()
boolfilter_nested.must(Terms(field='child_entities.entitytypeid', terms=[entitytype.pk]))
boolfilter_nested.must(Match(field='child_entities.value', query=term['value'], type='phrase'))
nested = Nested(path='child_entities', query=boolfilter_nested)
if term['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
elif term['type'] == 'concept':
concept_ids = _get_child_concepts(term['value'])
terms = Terms(field='domains.conceptid', terms=concept_ids)
nested = Nested(path='domains', query=terms)
if term['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
elif term['type'] == 'string':
boolfilter_folded = Bool()
boolfilter_folded.should(Match(field='child_entities.value', query=term['value'], type='phrase_prefix'))
boolfilter_folded.should(Match(field='child_entities.value.folded', query=term['value'], type='phrase_prefix'))
nested = Nested(path='child_entities', query=boolfilter_folded)
if term['inverted']:
boolquery.must_not(nested)
else:
boolquery.must(nested)
if 'geometry' in spatial_filter and 'type' in spatial_filter['geometry'] and spatial_filter['geometry']['type'] != '':
geojson = spatial_filter['geometry']
if geojson['type'] == 'bbox':
coordinates = [[geojson['coordinates'][0],geojson['coordinates'][3]], [geojson['coordinates'][2],geojson['coordinates'][1]]]
geoshape = GeoShape(field='geometries.value', type='envelope', coordinates=coordinates )
nested = Nested(path='geometries', query=geoshape)
else:
buffer = spatial_filter['buffer']
geojson = JSONDeserializer().deserialize(_buffer(geojson,buffer['width'],buffer['unit']).json)
geoshape = GeoShape(field='geometries.value', type=geojson['type'], coordinates=geojson['coordinates'] )
nested = Nested(path='geometries', query=geoshape)
if 'inverted' not in spatial_filter:
spatial_filter['inverted'] = False
if spatial_filter['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
if 'year_min_max' in temporal_filter and len(temporal_filter['year_min_max']) == 2:
start_date = date(temporal_filter['year_min_max'][0], 1, 1)
end_date = date(temporal_filter['year_min_max'][1], 12, 31)
if start_date:
start_date = start_date.isoformat()
if end_date:
end_date = end_date.isoformat()
range = Range(field='dates.value', gte=start_date, lte=end_date)
nested = Nested(path='dates', query=range)
if 'inverted' not in temporal_filter:
temporal_filter['inverted'] = False
if temporal_filter['inverted']:
boolfilter.must_not(nested)
else:
boolfilter.must(nested)
if not boolquery.empty:
query.add_query(boolquery)
if not boolfilter.empty:
query.add_filter(boolfilter)
return query
