def search_terms(request):
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
se = SearchEngineFactory().create()
searchString = request.GET.get('q', '')
user_is_reviewer = request.user.groups.filter(name='Resource Reviewer').exists()
i = 0
ret = {}
for index in ['terms', 'concepts']:
query = Query(se, start=0, limit=0)
boolquery = Bool()
boolquery.should(Match(field='value', query=searchString.lower(), type='phrase_prefix'))
boolquery.should(Match(field='value.folded', query=searchString.lower(), type='phrase_prefix'))
boolquery.should(Match(field='value.folded', query=searchString.lower(), fuzziness='AUTO', prefix_length=settings.SEARCH_TERM_SENSITIVITY))
if user_is_reviewer is False and index == 'terms':
boolquery.filter(Terms(field='provisional', terms=['false']))
query.add_query(boolquery)
base_agg = Aggregation(name='value_agg', type='terms', field='value.raw', size=settings.SEARCH_DROPDOWN_LENGTH, order={"max_score": "desc"})
nodegroupid_agg = Aggregation(name='nodegroupid', type='terms', field='nodegroupid')
top_concept_agg = Aggregation(name='top_concept', type='terms', field='top_concept')
conceptid_agg = Aggregation(name='conceptid', type='terms', field='conceptid')
max_score_agg = MaxAgg(name='max_score', script='_score')
top_concept_agg.add_aggregation(conceptid_agg)
base_agg.add_aggregation(max_score_agg)
base_agg.add_aggregation(top_concept_agg)
base_agg.add_aggregation(nodegroupid_agg)
query.add_aggregation(base_agg)
ret[index] = []
results = query.search(index=index)
for result in results['aggregations']['value_agg']['buckets']:
if len(result['top_concept']['buckets']) > 0:
for top_concept in result['top_concept']['buckets']:
top_concept_id = top_concept['key']
top_concept_label = get_preflabel_from_conceptid(top_concept['key'], lang)['value']
for concept in top_concept['conceptid']['buckets']:
ret[index].append({
'type': 'concept',
'context': top_concept_id,
'context_label': top_concept_label,
'id': i,
'text': result['key'],
'value': concept['key']
})
i = i + 1
else:
ret[index].append({
'type': 'term',
'context': '',
'context_label': get_resource_model_label(result),
'id': i,
'text': result['key'],
'value': result['key']
})
i = i + 1
return JSONResponse(ret)
def get_preflabel_from_conceptid(conceptid, lang):
ret = None
default = {
"category": "",
"conceptid": "",
"language": "",
"value": "",
"type": "",
"id": ""
}
query = Query(se)
bool_query = Bool()
bool_query.must(Match(field="type", query="prefLabel", type="phrase"))
bool_query.filter(Terms(field="conceptid", terms=[conceptid]))
query.add_query(bool_query)
preflabels = query.search(index=CONCEPTS_INDEX)["hits"]["hits"]
for preflabel in preflabels:
default = preflabel["_source"]
# get the label in the preferred language, otherwise get the label in the default language
if preflabel["_source"]["language"] == lang:
return preflabel["_source"]
if preflabel["_source"]["language"].split("-")[0] == lang.split(
"-")[0]:
ret = preflabel["_source"]
if preflabel["_source"][
"language"] == settings.LANGUAGE_CODE and ret is None:
ret = preflabel["_source"]
return default if ret is None else ret
def get_preflabel_from_conceptid(conceptid, lang):
ret = None
default = {
"category": "",
"conceptid": "",
"language": "",
"value": "",
"type": "",
"id": ""
}
se = SearchEngineFactory().create()
query = Query(se)
bool_query = Bool()
bool_query.must(Match(field='type', query='prefLabel', type='phrase'))
bool_query.filter(Terms(field='conceptid', terms=[conceptid]))
query.add_query(bool_query)
preflabels = query.search(index='strings', doc_type='concept')['hits']['hits']
for preflabel in preflabels:
default = preflabel['_source']
# get the label in the preferred language, otherwise get the label in the default language
if preflabel['_source']['language'] == lang:
return preflabel['_source']
if preflabel['_source']['language'].split('-')[0] == lang.split('-')[0]:
ret = preflabel['_source']
if preflabel['_source']['language'] == settings.LANGUAGE_CODE and ret == None:
ret = preflabel['_source']
return default if ret == None else ret
def delete(self, user={}, note=''):
"""
Deletes a single resource and any related indexed data
"""
se = SearchEngineFactory().create()
related_resources = self.get_related_resources(lang="en-US",
start=0,
limit=1000)
for rr in related_resources['resource_relationships']:
models.ResourceXResource.objects.get(pk=rr['resourcexid']).delete()
query = Query(se)
bool_query = Bool()
bool_query.filter(
Terms(field='resourceinstanceid', terms=[self.resourceinstanceid]))
query.add_query(bool_query)
results = query.search(index='strings',
doc_type='term')['hits']['hits']
for result in results:
se.delete(index='strings', doc_type='term', id=result['_id'])
se.delete(index='resource',
doc_type=str(self.graph_id),
id=self.resourceinstanceid)
self.save_edit(edit_type='delete')
super(Resource, self).delete()
def get_preflabel_from_conceptid(conceptid, lang):
ret = None
default = {
"category": "",
"conceptid": "",
"language": "",
"value": "",
"type": "",
"id": ""
}
se = SearchEngineFactory().create()
query = Query(se)
bool_query = Bool()
bool_query.must(Match(field='type', query='prefLabel', type='phrase'))
bool_query.filter(Terms(field='conceptid', terms=[conceptid]))
query.add_query(bool_query)
preflabels = query.search(index='strings', doc_type='concept')['hits']['hits']
for preflabel in preflabels:
default = preflabel['_source']
# get the label in the preferred language, otherwise get the label in the default language
if preflabel['_source']['language'] == lang:
return preflabel['_source']
if preflabel['_source']['language'].split('-')[0] == lang.split('-')[0]:
ret = preflabel['_source']
if preflabel['_source']['language'] == settings.LANGUAGE_CODE and ret == None:
ret = preflabel['_source']
return default if ret == None else ret
def delete(self, *args, **kwargs):
se = SearchEngineFactory().create()
request = kwargs.pop('request', None)
provisional_edit_log_details = kwargs.pop('provisional_edit_log_details', None)
for tile in self.tiles:
tile.delete(*args, request=request, **kwargs)
try:
user = request.user
user_is_reviewer = request.user.groups.filter(name='Resource Reviewer').exists()
except AttributeError: #no user
user = None
if user_is_reviewer is True or self.user_owns_provisional(user):
query = Query(se)
bool_query = Bool()
bool_query.filter(Terms(field='tileid', terms=[self.tileid]))
query.add_query(bool_query)
results = query.search(index='terms')['hits']['hits']
for result in results:
se.delete(index='terms', id=result['_id'])
self.__preDelete(request)
self.save_edit(
user=request.user,
edit_type='tile delete',
old_value=self.data,
provisional_edit_log_details=provisional_edit_log_details)
super(Tile, self).delete(*args, **kwargs)
resource = Resource.objects.get(resourceinstanceid=self.resourceinstance.resourceinstanceid)
resource.index()
else:
self.apply_provisional_edit(user, data={}, action='delete')
super(Tile, self).save(*args, **kwargs)
def delete(self, *args, **kwargs):
se = SearchEngineFactory().create()
request = kwargs.pop("request", None)
provisional_edit_log_details = kwargs.pop("provisional_edit_log_details", None)
for tile in self.tiles:
tile.delete(*args, request=request, **kwargs)
try:
user = request.user
user_is_reviewer = user_is_resource_reviewer(user)
except AttributeError: # no user
user = None
user_is_reviewer = True
if user_is_reviewer is True or self.user_owns_provisional(user):
query = Query(se)
bool_query = Bool()
bool_query.filter(Terms(field="tileid", terms=[self.tileid]))
query.add_query(bool_query)
results = query.search(index="terms")["hits"]["hits"]
for result in results:
se.delete(index="terms", id=result["_id"])
self.__preDelete(request)
self.save_edit(
user=request.user, edit_type="tile delete", old_value=self.data, provisional_edit_log_details=provisional_edit_log_details
)
super(Tile, self).delete(*args, **kwargs)
resource = Resource.objects.get(resourceinstanceid=self.resourceinstance.resourceinstanceid)
resource.index()
else:
self.apply_provisional_edit(user, data={}, action="delete")
super(Tile, self).save(*args, **kwargs)
def append_dsl(self, search_results_object, permitted_nodegroups,
include_provisional):
search_query = Bool()
querysting_params = self.request.GET.get(details["componentname"], "")
spatial_filter = JSONDeserializer().deserialize(querysting_params)
if "features" in spatial_filter:
if len(spatial_filter["features"]) > 0:
feature_geom = spatial_filter["features"][0]["geometry"]
feature_properties = {}
if "properties" in spatial_filter["features"][0]:
feature_properties = spatial_filter["features"][0][
"properties"]
buffer = {"width": 0, "unit": "ft"}
if "buffer" in feature_properties:
buffer = feature_properties["buffer"]
search_buffer = _buffer(feature_geom, buffer["width"],
buffer["unit"])
feature_geom = JSONDeserializer().deserialize(
search_buffer.geojson)
geoshape = GeoShape(field="geometries.geom.features.geometry",
type=feature_geom["type"],
coordinates=feature_geom["coordinates"])
invert_spatial_search = False
if "inverted" in feature_properties:
invert_spatial_search = feature_properties["inverted"]
spatial_query = Bool()
if invert_spatial_search is True:
spatial_query.must_not(geoshape)
else:
spatial_query.filter(geoshape)
# get the nodegroup_ids that the user has permission to search
spatial_query.filter(
Terms(field="geometries.nodegroup_id",
terms=permitted_nodegroups))
if include_provisional is False:
spatial_query.filter(
Terms(field="geometries.provisional", terms=["false"]))
elif include_provisional == "only provisional":
spatial_query.filter(
Terms(field="geometries.provisional", terms=["true"]))
search_query.filter(
Nested(path="geometries", query=spatial_query))
search_results_object["query"].add_query(search_query)
if details["componentname"] not in search_results_object:
search_results_object[details["componentname"]] = {}
try:
search_results_object[
details["componentname"]]["search_buffer"] = feature_geom
except NameError:
logger.info(_("Feature geometry is not defined"))
def create_nested_attribute_filter(self, doc_id, nodegroup_id, value_list):
new_string_filter = Bool()
new_string_filter.filter(Terms(field='strings.nodegroup_id', terms=[nodegroup_id]))
for value in value_list:
new_string_filter.should(Match(field='strings.string', query=value, type='phrase'))
nested = Nested(path='strings', query=new_string_filter)
return nested
def search_terms(request):
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
se = SearchEngineFactory().create()
searchString = request.GET.get('q', '')
query = Query(se, start=0, limit=0)
user_is_reviewer = request.user.groups.filter(name='Resource Reviewer').exists()
boolquery = Bool()
boolquery.should(Match(field='value', query=searchString.lower(), type='phrase_prefix', fuzziness='AUTO'))
boolquery.should(Match(field='value.folded', query=searchString.lower(), type='phrase_prefix', fuzziness='AUTO'))
boolquery.should(Match(field='value.folded', query=searchString.lower(), fuzziness='AUTO'))
if user_is_reviewer is False:
boolquery.filter(Terms(field='provisional', terms=['false']))
query.add_query(boolquery)
base_agg = Aggregation(name='value_agg', type='terms', field='value.raw', size=settings.SEARCH_DROPDOWN_LENGTH, order={"max_score": "desc"})
nodegroupid_agg = Aggregation(name='nodegroupid', type='terms', field='nodegroupid')
top_concept_agg = Aggregation(name='top_concept', type='terms', field='top_concept')
conceptid_agg = Aggregation(name='conceptid', type='terms', field='conceptid')
max_score_agg = MaxAgg(name='max_score', script='_score')
top_concept_agg.add_aggregation(conceptid_agg)
base_agg.add_aggregation(max_score_agg)
base_agg.add_aggregation(top_concept_agg)
base_agg.add_aggregation(nodegroupid_agg)
query.add_aggregation(base_agg)
results = query.search(index='strings') or {'hits': {'hits':[]}}
i = 0;
ret = []
for result in results['aggregations']['value_agg']['buckets']:
if len(result['top_concept']['buckets']) > 0:
for top_concept in result['top_concept']['buckets']:
top_concept_id = top_concept['key']
top_concept_label = get_preflabel_from_conceptid(top_concept['key'], lang)['value']
for concept in top_concept['conceptid']['buckets']:
ret.append({
'type': 'concept',
'context': top_concept_id,
'context_label': top_concept_label,
'id': i,
'text': result['key'],
'value': concept['key']
})
i = i + 1
else:
ret.append({
'type': 'term',
'context': '',
'context_label': get_resource_model_label(result),
'id': i,
'text': result['key'],
'value': result['key']
})
i = i + 1
return JSONResponse(ret)
def delete(self, user={}, note=""):
"""
Deletes a single resource and any related indexed data
"""
permit_deletion = False
graph = models.GraphModel.objects.get(graphid=self.graph_id)
if graph.isactive is False:
message = _("This model is not yet active; unable to delete.")
raise ModelInactiveError(message)
if user != {}:
user_is_reviewer = user_is_resource_reviewer(user)
if user_is_reviewer is False:
tiles = list(models.TileModel.objects.filter(resourceinstance=self))
resource_is_provisional = True if sum([len(t.data) for t in tiles]) == 0 else False
if resource_is_provisional is True:
permit_deletion = True
else:
permit_deletion = True
else:
permit_deletion = True
if permit_deletion is True:
related_resources = self.get_related_resources(lang="en-US", start=0, limit=1000, page=0)
for rr in related_resources["resource_relationships"]:
# delete any related resource entries, also reindex the resource that references this resource that's being deleted
try:
resourceXresource = models.ResourceXResource.objects.get(pk=rr["resourcexid"])
resource_to_reindex = (
resourceXresource.resourceinstanceidfrom_id
if resourceXresource.resourceinstanceidto_id == self.resourceinstanceid
else resourceXresource.resourceinstanceidto_id
)
resourceXresource.delete(deletedResourceId=self.resourceinstanceid)
res = Resource.objects.get(pk=resource_to_reindex)
res.load_tiles()
res.index()
except ObjectDoesNotExist:
se.delete(index=RESOURCE_RELATIONS_INDEX, id=rr["resourcexid"])
query = Query(se)
bool_query = Bool()
bool_query.filter(Terms(field="resourceinstanceid", terms=[self.resourceinstanceid]))
query.add_query(bool_query)
results = query.search(index=TERMS_INDEX)["hits"]["hits"]
for result in results:
se.delete(index=TERMS_INDEX, id=result["_id"])
se.delete(index=RESOURCES_INDEX, id=self.resourceinstanceid)
try:
self.save_edit(edit_type="delete", user=user, note=self.displayname)
except:
pass
super(Resource, self).delete()
return permit_deletion
def append_dsl(self, search_results_object, permitted_nodegroups,
include_provisional):
search_query = Bool()
querysting_params = self.request.GET.get(details['componentname'], '')
spatial_filter = JSONDeserializer().deserialize(querysting_params)
if 'features' in spatial_filter:
if len(spatial_filter['features']) > 0:
feature_geom = spatial_filter['features'][0]['geometry']
feature_properties = {}
if 'properties' in spatial_filter['features'][0]:
feature_properties = spatial_filter['features'][0][
'properties']
buffer = {'width': 0, 'unit': 'ft'}
if 'buffer' in feature_properties:
buffer = feature_properties['buffer']
search_buffer = _buffer(feature_geom, buffer['width'],
buffer['unit'])
feature_geom = JSONDeserializer().deserialize(
search_buffer.json)
geoshape = GeoShape(field='geometries.geom.features.geometry',
type=feature_geom['type'],
coordinates=feature_geom['coordinates'])
invert_spatial_search = False
if 'inverted' in feature_properties:
invert_spatial_search = feature_properties['inverted']
spatial_query = Bool()
if invert_spatial_search is True:
spatial_query.must_not(geoshape)
else:
spatial_query.filter(geoshape)
# get the nodegroup_ids that the user has permission to search
spatial_query.filter(
Terms(field='geometries.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional is False:
spatial_query.filter(
Terms(field='geometries.provisional', terms=['false']))
elif include_provisional == 'only provisional':
spatial_query.filter(
Terms(field='geometries.provisional', terms=['true']))
search_query.filter(
Nested(path='geometries', query=spatial_query))
search_results_object['query'].add_query(search_query)
if details['componentname'] not in search_results_object:
search_results_object[details['componentname']] = {}
search_results_object[
details['componentname']]['search_buffer'] = search_buffer.geojson
def append_null_search_filters(self, value, node, query, request):
"""
Appends the search query dsl to search for fields that have not been populated
"""
base_query = Bool()
base_query.filter(Terms(field="graph_id", terms=[str(node.graph_id)]))
null_query = Bool()
data_exists_query = Exists(field="tiles.data.%s" % (str(node.pk)))
nested_query = Nested(path="tiles", query=data_exists_query)
null_query.must(nested_query)
if value["op"] == "null":
# search for tiles that don't exist
exists_query = Bool()
exists_query.must_not(null_query)
base_query.should(exists_query)
# search for tiles that do exist, but that have null or [] as values
func_query = Dsl()
func_query.dsl = {
"function_score": {
"min_score": 1,
"query": {"match_all": {}},
"functions": [
{
"script_score": {
"script": {
"source": """
int null_docs = 0;
for(tile in params._source.tiles){
if(tile.data.containsKey(params.node_id)){
def val = tile.data.get(params.node_id);
if (val == null || (val instanceof List && val.length==0)) {
null_docs++;
break;
}
}
}
return null_docs;
""",
"lang": "painless",
"params": {"node_id": "%s" % (str(node.pk))},
}
}
}
],
"score_mode": "max",
"boost": 1,
"boost_mode": "replace",
}
}
base_query.should(func_query)
elif value["op"] == "not_null":
base_query.must(null_query)
query.must(base_query)
def delete_index(self, resourceinstanceid=None):
"""
Deletes all references to a resource from all indexes
Keyword Arguments:
resourceinstanceid -- the resource instance id to delete from related indexes, if supplied will use this over self.resourceinstanceid
"""
if resourceinstanceid is None:
resourceinstanceid = self.resourceinstanceid
resourceinstanceid = str(resourceinstanceid)
# delete any related terms
query = Query(se)
bool_query = Bool()
bool_query.filter(
Terms(field="resourceinstanceid", terms=[resourceinstanceid]))
query.add_query(bool_query)
query.delete(index=TERMS_INDEX)
# delete any related resource index entries
query = Query(se)
bool_query = Bool()
bool_query.should(
Terms(field="resourceinstanceidto", terms=[resourceinstanceid]))
bool_query.should(
Terms(field="resourceinstanceidfrom", terms=[resourceinstanceid]))
query.add_query(bool_query)
query.delete(index=RESOURCE_RELATIONS_INDEX)
# reindex any related resources
query = Query(se)
bool_query = Bool()
bool_query.filter(
Nested(path="ids",
query=Terms(field="ids.id", terms=[resourceinstanceid])))
query.add_query(bool_query)
results = query.search(index=RESOURCES_INDEX)["hits"]["hits"]
for result in results:
try:
res = Resource.objects.get(pk=result["_id"])
res.load_tiles()
res.index()
except ObjectDoesNotExist:
pass
# delete resource index
se.delete(index=RESOURCES_INDEX, id=resourceinstanceid)
# delete resources from custom indexes
for index in settings.ELASTICSEARCH_CUSTOM_INDEXES:
es_index = import_class_from_string(index["module"])(index["name"])
es_index.delete_resources(resources=self)
def delete(self, user={}, note=''):
"""
Deletes a single resource and any related indexed data
"""
permit_deletion = False
graph = models.GraphModel.objects.get(graphid=self.graph_id)
if graph.isactive is False:
message = _('This model is not yet active; unable to delete.')
raise ModelInactiveError(message)
if user != {}:
user_is_reviewer = user.groups.filter(
name='Resource Reviewer').exists()
if user_is_reviewer is False:
tiles = list(
models.TileModel.objects.filter(resourceinstance=self))
resource_is_provisional = True if sum(
[len(t.data) for t in tiles]) == 0 else False
if resource_is_provisional is True:
permit_deletion = True
else:
permit_deletion = True
else:
permit_deletion = True
if permit_deletion is True:
se = SearchEngineFactory().create()
related_resources = self.get_related_resources(lang="en-US",
start=0,
limit=1000,
page=0)
for rr in related_resources['resource_relationships']:
models.ResourceXResource.objects.get(
pk=rr['resourcexid']).delete()
query = Query(se)
bool_query = Bool()
bool_query.filter(
Terms(field='resourceinstanceid',
terms=[self.resourceinstanceid]))
query.add_query(bool_query)
results = query.search(index='terms')['hits']['hits']
for result in results:
se.delete(index='terms', id=result['_id'])
se.delete(index='resources', id=self.resourceinstanceid)
self.save_edit(edit_type='delete',
user=user,
note=self.displayname)
super(Resource, self).delete()
return permit_deletion
def append_dsl(self, search_results_object, permitted_nodegroups,
include_provisional):
nested_agg = NestedAgg(path='points', name='geo_aggs')
nested_agg_filter = FiltersAgg(name='inner')
geo_agg_filter = Bool()
if include_provisional is True:
geo_agg_filter.filter(
Terms(field='points.provisional', terms=['false', 'true']))
else:
if include_provisional is False:
geo_agg_filter.filter(
Terms(field='points.provisional', terms=['false']))
elif include_provisional is 'only provisional':
geo_agg_filter.filter(
Terms(field='points.provisional', terms=['true']))
geo_agg_filter.filter(
Terms(field='points.nodegroup_id', terms=permitted_nodegroups))
nested_agg_filter.add_filter(geo_agg_filter)
nested_agg_filter.add_aggregation(
GeoHashGridAgg(field='points.point',
name='grid',
precision=settings.HEX_BIN_PRECISION))
nested_agg_filter.add_aggregation(
GeoBoundsAgg(field='points.point', name='bounds'))
nested_agg.add_aggregation(nested_agg_filter)
search_results_object['query'].add_aggregation(nested_agg)
def create_nested_geo_filter(self, geometry):
## process GEOS geometry object into geojson and create ES filter
geojson_geom = JSONDeserializer().deserialize(geometry.geojson)
geoshape = GeoShape(
field="geometries.geom.features.geometry",
type=geojson_geom["type"],
coordinates=geojson_geom["coordinates"]
)
new_spatial_filter = Bool()
new_spatial_filter.filter(geoshape)
nested = Nested(path='geometries', query=new_spatial_filter)
return nested
def gen_range_agg(gte=None, lte=None, permitted_nodegroups=None):
date_query = Bool()
date_query.filter(
Range(field="dates.date",
gte=gte,
lte=lte,
relation="intersects"))
if permitted_nodegroups is not None:
date_query.filter(
Terms(field="dates.nodegroup_id",
terms=permitted_nodegroups))
date_ranges_query = Bool()
date_ranges_query.filter(
Range(field="date_ranges.date_range",
gte=gte,
lte=lte,
relation="intersects"))
if permitted_nodegroups is not None:
date_ranges_query.filter(
Terms(field="date_ranges.nodegroup_id",
terms=permitted_nodegroups))
wrapper_query = Bool()
wrapper_query.should(
Nested(path="date_ranges", query=date_ranges_query))
wrapper_query.should(Nested(path="dates", query=date_query))
return wrapper_query
def gen_range_agg(gte=None, lte=None, permitted_nodegroups=None):
date_query = Bool()
date_query.filter(
Range(field='dates.date',
gte=gte,
lte=lte,
relation='intersects'))
if permitted_nodegroups:
date_query.filter(
Terms(field='dates.nodegroup_id',
terms=permitted_nodegroups))
date_ranges_query = Bool()
date_ranges_query.filter(
Range(field='date_ranges.date_range',
gte=gte,
lte=lte,
relation='intersects'))
if permitted_nodegroups:
date_ranges_query.filter(
Terms(field='date_ranges.nodegroup_id',
terms=permitted_nodegroups))
wrapper_query = Bool()
wrapper_query.should(
Nested(path='date_ranges', query=date_ranges_query))
wrapper_query.should(Nested(path='dates', query=date_query))
return wrapper_query
def append_dsl(self, search_results_object, permitted_nodegroups,
include_provisional):
nested_agg = NestedAgg(path="points", name="geo_aggs")
nested_agg_filter = FiltersAgg(name="inner")
geo_agg_filter = Bool()
if include_provisional is True:
geo_agg_filter.filter(
Terms(field="points.provisional", terms=["false", "true"]))
else:
if include_provisional is False:
geo_agg_filter.filter(
Terms(field="points.provisional", terms=["false"]))
elif include_provisional is "only provisional":
geo_agg_filter.filter(
Terms(field="points.provisional", terms=["true"]))
geo_agg_filter.filter(
Terms(field="points.nodegroup_id", terms=permitted_nodegroups))
nested_agg_filter.add_filter(geo_agg_filter)
nested_agg_filter.add_aggregation(
GeoHashGridAgg(field="points.point",
name="grid",
precision=settings.HEX_BIN_PRECISION))
nested_agg_filter.add_aggregation(
GeoBoundsAgg(field="points.point", name="bounds"))
nested_agg.add_aggregation(nested_agg_filter)
search_results_object["query"].add_aggregation(nested_agg)
def append_null_search_filters(self, value, node, query, request):
"""
Appends the search query dsl to search for fields that haven't been populated
"""
base_query = Bool()
null_query = Bool()
data_exists_query = Exists(field="tiles.data.%s" % (str(node.pk)))
nested_query = Nested(path="tiles", query=data_exists_query)
null_query.must(nested_query)
base_query.filter(Terms(field="graph_id", terms=[str(node.graph_id)]))
if value["op"] == "null":
base_query.must_not(null_query)
elif value["op"] == "not_null":
base_query.must(null_query)
query.must(base_query)
def delete(self, *args, **kwargs):
se = SearchEngineFactory().create()
request = kwargs.pop("request", None)
index = kwargs.pop("index", True)
transaction_id = kwargs.pop("index", None)
provisional_edit_log_details = kwargs.pop(
"provisional_edit_log_details", None)
for tile in self.tiles:
tile.delete(*args, request=request, **kwargs)
try:
user = request.user
user_is_reviewer = user_is_resource_reviewer(user)
except AttributeError: # no user
user = None
user_is_reviewer = True
if user_is_reviewer is True or self.user_owns_provisional(user):
if index:
query = Query(se)
bool_query = Bool()
bool_query.filter(Terms(field="tileid", terms=[self.tileid]))
query.add_query(bool_query)
results = query.delete(index=TERMS_INDEX)
self.__preDelete(request)
self.save_edit(
user=user,
edit_type="tile delete",
old_value=self.data,
provisional_edit_log_details=provisional_edit_log_details,
transaction_id=transaction_id,
)
try:
super(Tile, self).delete(*args, **kwargs)
for nodeid in self.data.keys():
node = models.Node.objects.get(nodeid=nodeid)
datatype = self.datatype_factory.get_instance(
node.datatype)
datatype.post_tile_delete(self, nodeid, index=index)
if index:
self.index()
except IntegrityError as e:
logger.error(e)
else:
self.apply_provisional_edit(user, data={}, action="delete")
super(Tile, self).save(*args, **kwargs)
def append_dsl(self, search_results_object, permitted_nodegroups,
include_provisional):
search_query = Bool()
querysting_params = self.request.GET.get(details["componentname"], "")
graph_ids = []
for resouceTypeFilter in JSONDeserializer().deserialize(
querysting_params):
graph_ids.append(str(resouceTypeFilter["graphid"]))
terms = Terms(field="graph_id", terms=graph_ids)
if resouceTypeFilter["inverted"] is True:
search_query.must_not(terms)
else:
search_query.filter(terms)
search_results_object["query"].add_query(search_query)
def append_dsl(self, search_results_object, permitted_nodegroups,
include_provisional):
search_query = Bool()
if include_provisional is not True:
provisional_resource_filter = Bool()
if include_provisional is False:
provisional_resource_filter.filter(
Terms(field='provisional_resource',
terms=['false', 'partial']))
elif include_provisional is 'only provisional':
provisional_resource_filter.filter(
Terms(field='provisional_resource',
terms=['true', 'partial']))
search_query.must(provisional_resource_filter)
search_results_object['query'].add_query(search_query)
def append_dsl(self, search_results_object, permitted_nodegroups,
include_provisional):
search_query = Bool()
if include_provisional is not True:
provisional_resource_filter = Bool()
if include_provisional is False:
provisional_resource_filter.filter(
Terms(field="provisional_resource",
terms=["false", "partial"]))
elif include_provisional == "only provisional":
provisional_resource_filter.filter(
Terms(field="provisional_resource",
terms=["true", "partial"]))
search_query.must(provisional_resource_filter)
search_results_object["query"].add_query(search_query)
def delete(self, *args, **kwargs):
se = SearchEngineFactory().create()
request = kwargs.pop('request', None)
for tiles in self.tiles.itervalues():
for tile in tiles:
tile.delete(*args, request=request, **kwargs)
query = Query(se)
bool_query = Bool()
bool_query.filter(Terms(field='tileid', terms=[self.tileid]))
query.add_query(bool_query)
results = query.search(index='strings', doc_type='term')['hits']['hits']
for result in results:
se.delete(index='strings', doc_type='term', id=result['_id'])
self.__preDelete(request)
super(Tile, self).delete(*args, **kwargs)
resource = Resource.objects.get(resourceinstanceid=self.resourceinstance.resourceinstanceid)
resource.index()
def delete(self):
"""
Deletes a single resource and any related indexed data
"""
se = SearchEngineFactory().create()
related_resources = self.get_related_resources(lang="en-US", start=0, limit=15)
for rr in related_resources['resource_relationships']:
models.ResourceXResource.objects.get(pk=rr['resourcexid']).delete()
query = Query(se)
bool_query = Bool()
bool_query.filter(Terms(field='resourceinstanceid', terms=[self.resourceinstanceid]))
query.add_query(bool_query)
results = query.search(index='strings', doc_type='term')['hits']['hits']
for result in results:
se.delete(index='strings', doc_type='term', id=result['_id'])
se.delete(index='resource', doc_type=str(self.graph_id), id=self.resourceinstanceid)
super(Resource, self).delete()
def delete(self, *args, **kwargs):
se = SearchEngineFactory().create()
request = kwargs.pop('request', None)
for tiles in self.tiles.itervalues():
for tile in tiles:
tile.delete(*args, request=request, **kwargs)
query = Query(se)
bool_query = Bool()
bool_query.filter(Terms(field='tileid', terms=[self.tileid]))
query.add_query(bool_query)
results = query.search(index='strings', doc_type='term')['hits']['hits']
for result in results:
se.delete(index='strings', doc_type='term', id=result['_id'])
self.__preDelete(request)
self.save_edit(user=request.user, edit_type='tile delete', old_value=self.data)
super(Tile, self).delete(*args, **kwargs)
resource = Resource.objects.get(resourceinstanceid=self.resourceinstance.resourceinstanceid)
resource.index()
def gen_range_agg(gte=None, lte=None, permitted_nodegroups=None):
date_query = Bool()
date_query.filter(Range(field='dates.date', gte=gte, lte=lte, relation='intersects'))
if permitted_nodegroups:
date_query.filter(Terms(field='dates.nodegroup_id', terms=permitted_nodegroups))
date_ranges_query = Bool()
date_ranges_query.filter(Range(field='date_ranges.date_range', gte=gte, lte=lte, relation='intersects'))
if permitted_nodegroups:
date_ranges_query.filter(Terms(field='date_ranges.nodegroup_id', terms=permitted_nodegroups))
wrapper_query = Bool()
wrapper_query.should(Nested(path='date_ranges', query=date_ranges_query))
wrapper_query.should(Nested(path='dates', query=date_query))
return wrapper_query
def delete_index(self, resourceinstanceid=None):
"""
Deletes all references to a resource from all indexes
Keyword Arguments:
resourceinstanceid -- the resource instance id to delete from related indexes, if supplied will use this over self.resourceinstanceid
"""
if resourceinstanceid is None:
resourceinstanceid = self.resourceinstanceid
resourceinstanceid = str(resourceinstanceid)
# delete any related terms
query = Query(se)
bool_query = Bool()
bool_query.filter(Terms(field="resourceinstanceid", terms=[resourceinstanceid]))
query.add_query(bool_query)
query.delete(index=TERMS_INDEX)
# delete any related resource index entries
query = Query(se)
bool_query = Bool()
bool_query.should(Terms(field="resourceinstanceidto", terms=[resourceinstanceid]))
bool_query.should(Terms(field="resourceinstanceidfrom", terms=[resourceinstanceid]))
query.add_query(bool_query)
query.delete(index=RESOURCE_RELATIONS_INDEX)
# reindex any related resources
query = Query(se)
bool_query = Bool()
bool_query.filter(Nested(path="ids", query=Terms(field="ids.id", terms=[resourceinstanceid])))
query.add_query(bool_query)
results = query.search(index=RESOURCES_INDEX)["hits"]["hits"]
for result in results:
res = Resource.objects.get(pk=result["_id"])
res.load_tiles()
res.index()
# delete resource index
se.delete(index=RESOURCES_INDEX, id=resourceinstanceid)
def append_search_filters(self, value, node, query, request):
try:
if value["op"] == "null" or value["op"] == "not_null":
self.append_null_search_filters(value, node, query, request)
elif value["val"] != "":
base_query = Bool()
base_query.filter(
Terms(field="graph_id", terms=[str(node.graph_id)]))
match_query = Nested(path="tiles",
query=Match(field="tiles.data.%s" %
(str(node.pk)),
type="phrase",
query=value["val"]))
if "!" in value["op"]:
base_query.must_not(match_query)
# base_query.filter(Exists(field="tiles.data.%s" % (str(node.pk))))
else:
base_query.must(match_query)
query.must(base_query)
except KeyError as e:
pass
def get_relations(resourceinstanceid,
start,
limit,
resourceinstance_graphid=None):
query = Query(se, start=start, limit=limit)
bool_filter = Bool()
bool_filter.should(
Terms(field="resourceinstanceidfrom",
terms=resourceinstanceid))
bool_filter.should(
Terms(field="resourceinstanceidto", terms=resourceinstanceid))
if resourceinstance_graphid:
graph_filter = Bool()
to_graph_id_filter = Bool()
to_graph_id_filter.filter(
Terms(field="resourceinstancefrom_graphid",
terms=str(self.graph_id)))
to_graph_id_filter.filter(
Terms(field="resourceinstanceto_graphid",
terms=resourceinstance_graphid))
graph_filter.should(to_graph_id_filter)
from_graph_id_filter = Bool()
from_graph_id_filter.filter(
Terms(field="resourceinstancefrom_graphid",
terms=resourceinstance_graphid))
from_graph_id_filter.filter(
Terms(field="resourceinstanceto_graphid",
terms=str(self.graph_id)))
graph_filter.should(from_graph_id_filter)
bool_filter.must(graph_filter)
query.add_query(bool_filter)
return query.search(index=RESOURCE_RELATIONS_INDEX)
def search_terms(request):
lang = request.GET.get("lang", settings.LANGUAGE_CODE)
se = SearchEngineFactory().create()
searchString = request.GET.get("q", "")
user_is_reviewer = user_is_resource_reviewer(request.user)
i = 0
ret = {}
for index in ["terms", "concepts"]:
query = Query(se, start=0, limit=0)
boolquery = Bool()
boolquery.should(
Match(field="value",
query=searchString.lower(),
type="phrase_prefix"))
boolquery.should(
Match(field="value.folded",
query=searchString.lower(),
type="phrase_prefix"))
boolquery.should(
Match(field="value.folded",
query=searchString.lower(),
fuzziness="AUTO",
prefix_length=settings.SEARCH_TERM_SENSITIVITY))
if user_is_reviewer is False and index == "terms":
boolquery.filter(Terms(field="provisional", terms=["false"]))
query.add_query(boolquery)
base_agg = Aggregation(name="value_agg",
type="terms",
field="value.raw",
size=settings.SEARCH_DROPDOWN_LENGTH,
order={"max_score": "desc"})
nodegroupid_agg = Aggregation(name="nodegroupid",
type="terms",
field="nodegroupid")
top_concept_agg = Aggregation(name="top_concept",
type="terms",
field="top_concept")
conceptid_agg = Aggregation(name="conceptid",
type="terms",
field="conceptid")
max_score_agg = MaxAgg(name="max_score", script="_score")
top_concept_agg.add_aggregation(conceptid_agg)
base_agg.add_aggregation(max_score_agg)
base_agg.add_aggregation(top_concept_agg)
base_agg.add_aggregation(nodegroupid_agg)
query.add_aggregation(base_agg)
ret[index] = []
results = query.search(index=index)
if results is not None:
for result in results["aggregations"]["value_agg"]["buckets"]:
if len(result["top_concept"]["buckets"]) > 0:
for top_concept in result["top_concept"]["buckets"]:
top_concept_id = top_concept["key"]
top_concept_label = get_preflabel_from_conceptid(
top_concept["key"], lang)["value"]
for concept in top_concept["conceptid"]["buckets"]:
ret[index].append({
"type": "concept",
"context": top_concept_id,
"context_label": top_concept_label,
"id": i,
"text": result["key"],
"value": concept["key"],
})
i = i + 1
else:
ret[index].append({
"type":
"term",
"context":
"",
"context_label":
get_resource_model_label(result),
"id":
i,
"text":
result["key"],
"value":
result["key"],
})
i = i + 1
return JSONResponse(ret)
def build_search_results_dsl(request):
term_filter = request.GET.get('termFilter', '')
spatial_filter = JSONDeserializer().deserialize(
request.GET.get('mapFilter', '{}'))
include_provisional = get_provisional_type(request)
export = request.GET.get('export', None)
mobile_download = request.GET.get('mobiledownload', None)
page = 1 if request.GET.get('page') == '' else int(
request.GET.get('page', 1))
temporal_filter = JSONDeserializer().deserialize(
request.GET.get('temporalFilter', '{}'))
advanced_filters = JSONDeserializer().deserialize(
request.GET.get('advanced', '[]'))
search_buffer = None
se = SearchEngineFactory().create()
if export != None:
limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE
elif mobile_download != None:
limit = settings.MOBILE_DOWNLOAD_RESOURCE_LIMIT
else:
limit = settings.SEARCH_ITEMS_PER_PAGE
query = Query(se, start=limit * int(page - 1), limit=limit)
search_query = Bool()
nested_agg = NestedAgg(path='points', name='geo_aggs')
nested_agg_filter = FiltersAgg(name='inner')
if include_provisional == True:
nested_agg_filter.add_filter(
Terms(field='points.provisional', terms=['false', 'true']))
else:
provisional_resource_filter = Bool()
if include_provisional == False:
provisional_resource_filter.filter(
Terms(field='provisional', terms=['false', 'partial']))
nested_agg_filter.add_filter(
Terms(field='points.provisional', terms=['false']))
elif include_provisional == 'only provisional':
provisional_resource_filter.filter(
Terms(field='provisional', terms=['true', 'partial']))
nested_agg_filter.add_filter(
Terms(field='points.provisional', terms=['true']))
search_query.must(provisional_resource_filter)
nested_agg_filter.add_aggregation(
GeoHashGridAgg(field='points.point',
name='grid',
precision=settings.HEX_BIN_PRECISION))
nested_agg_filter.add_aggregation(
GeoBoundsAgg(field='points.point', name='bounds'))
nested_agg.add_aggregation(nested_agg_filter)
query.add_aggregation(nested_agg)
permitted_nodegroups = get_permitted_nodegroups(request.user)
if term_filter != '':
for term in JSONDeserializer().deserialize(term_filter):
term_query = Bool()
provisional_term_filter = Bool()
if term['type'] == 'term' or term['type'] == 'string':
string_filter = Bool()
if term['type'] == 'term':
string_filter.must(
Match(field='strings.string',
query=term['value'],
type='phrase'))
elif term['type'] == 'string':
string_filter.should(
Match(field='strings.string',
query=term['value'],
type='phrase_prefix'))
string_filter.should(
Match(field='strings.string.folded',
query=term['value'],
type='phrase_prefix'))
if include_provisional == False:
string_filter.must_not(
Match(field='strings.provisional',
query='true',
type='phrase'))
elif include_provisional == 'only provisional':
string_filter.must_not(
Match(field='strings.provisional',
query='false',
type='phrase'))
string_filter.filter(
Terms(field='strings.nodegroup_id',
terms=permitted_nodegroups))
nested_string_filter = Nested(path='strings',
query=string_filter)
if term['inverted']:
search_query.must_not(nested_string_filter)
else:
search_query.must(nested_string_filter)
# need to set min_score because the query returns results with score 0 and those have to be removed, which I don't think it should be doing
query.min_score('0.01')
elif term['type'] == 'concept':
concept_ids = _get_child_concepts(term['value'])
conceptid_filter = Bool()
conceptid_filter.filter(
Terms(field='domains.conceptid', terms=concept_ids))
conceptid_filter.filter(
Terms(field='domains.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional == False:
conceptid_filter.must_not(
Match(field='domains.provisional',
query='true',
type='phrase'))
elif include_provisional == 'only provisional':
conceptid_filter.must_not(
Match(field='domains.provisional',
query='false',
type='phrase'))
nested_conceptid_filter = Nested(path='domains',
query=conceptid_filter)
if term['inverted']:
search_query.must_not(nested_conceptid_filter)
else:
search_query.filter(nested_conceptid_filter)
if 'features' in spatial_filter:
if len(spatial_filter['features']) > 0:
feature_geom = spatial_filter['features'][0]['geometry']
feature_properties = {}
if 'properties' in spatial_filter['features'][0]:
feature_properties = spatial_filter['features'][0][
'properties']
buffer = {'width': 0, 'unit': 'ft'}
if 'buffer' in feature_properties:
buffer = feature_properties['buffer']
search_buffer = _buffer(feature_geom, buffer['width'],
buffer['unit'])
feature_geom = JSONDeserializer().deserialize(search_buffer.json)
geoshape = GeoShape(field='geometries.geom.features.geometry',
type=feature_geom['type'],
coordinates=feature_geom['coordinates'])
invert_spatial_search = False
if 'inverted' in feature_properties:
invert_spatial_search = feature_properties['inverted']
spatial_query = Bool()
if invert_spatial_search == True:
spatial_query.must_not(geoshape)
else:
spatial_query.filter(geoshape)
# get the nodegroup_ids that the user has permission to search
spatial_query.filter(
Terms(field='geometries.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional == False:
spatial_query.filter(
Terms(field='geometries.provisional', terms=['false']))
elif include_provisional == 'only provisional':
spatial_query.filter(
Terms(field='geometries.provisional', terms=['true']))
search_query.filter(Nested(path='geometries', query=spatial_query))
if 'fromDate' in temporal_filter and 'toDate' in temporal_filter:
now = str(datetime.utcnow())
start_date = ExtendedDateFormat(temporal_filter['fromDate'])
end_date = ExtendedDateFormat(temporal_filter['toDate'])
date_nodeid = str(
temporal_filter['dateNodeId']
) if 'dateNodeId' in temporal_filter and temporal_filter[
'dateNodeId'] != '' else None
query_inverted = False if 'inverted' not in temporal_filter else temporal_filter[
'inverted']
temporal_query = Bool()
if query_inverted:
# inverted date searches need to use an OR clause and are generally more complicated to structure (can't use ES must_not)
# eg: less than START_DATE OR greater than END_DATE
inverted_date_query = Bool()
inverted_date_ranges_query = Bool()
if start_date.is_valid():
inverted_date_query.should(
Range(field='dates.date', lt=start_date.lower))
inverted_date_ranges_query.should(
Range(field='date_ranges.date_range', lt=start_date.lower))
if end_date.is_valid():
inverted_date_query.should(
Range(field='dates.date', gt=end_date.upper))
inverted_date_ranges_query.should(
Range(field='date_ranges.date_range', gt=end_date.upper))
date_query = Bool()
date_query.filter(inverted_date_query)
date_query.filter(
Terms(field='dates.nodegroup_id', terms=permitted_nodegroups))
if include_provisional == False:
date_query.filter(
Terms(field='dates.provisional', terms=['false']))
elif include_provisional == 'only provisional':
date_query.filter(
Terms(field='dates.provisional', terms=['true']))
if date_nodeid:
date_query.filter(Term(field='dates.nodeid', term=date_nodeid))
else:
date_ranges_query = Bool()
date_ranges_query.filter(inverted_date_ranges_query)
date_ranges_query.filter(
Terms(field='date_ranges.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional == False:
date_ranges_query.filter(
Terms(field='date_ranges.provisional',
terms=['false']))
elif include_provisional == 'only provisional':
date_ranges_query.filter(
Terms(field='date_ranges.provisional', terms=['true']))
temporal_query.should(
Nested(path='date_ranges', query=date_ranges_query))
temporal_query.should(Nested(path='dates', query=date_query))
else:
date_query = Bool()
date_query.filter(
Range(field='dates.date',
gte=start_date.lower,
lte=end_date.upper))
date_query.filter(
Terms(field='dates.nodegroup_id', terms=permitted_nodegroups))
if include_provisional == False:
date_query.filter(
Terms(field='dates.provisional', terms=['false']))
elif include_provisional == 'only provisional':
date_query.filter(
Terms(field='dates.provisional', terms=['true']))
if date_nodeid:
date_query.filter(Term(field='dates.nodeid', term=date_nodeid))
else:
date_ranges_query = Bool()
date_ranges_query.filter(
Range(field='date_ranges.date_range',
gte=start_date.lower,
lte=end_date.upper,
relation='intersects'))
date_ranges_query.filter(
Terms(field='date_ranges.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional == False:
date_ranges_query.filter(
Terms(field='date_ranges.provisional',
terms=['false']))
if include_provisional == 'only provisional':
date_ranges_query.filter(
Terms(field='date_ranges.provisional', terms=['true']))
temporal_query.should(
Nested(path='date_ranges', query=date_ranges_query))
temporal_query.should(Nested(path='dates', query=date_query))
search_query.filter(temporal_query)
datatype_factory = DataTypeFactory()
if len(advanced_filters) > 0:
advanced_query = Bool()
grouped_query = Bool()
grouped_queries = [grouped_query]
for index, advanced_filter in enumerate(advanced_filters):
tile_query = Bool()
for key, val in advanced_filter.iteritems():
if key != 'op':
node = models.Node.objects.get(pk=key)
if request.user.has_perm('read_nodegroup', node.nodegroup):
datatype = datatype_factory.get_instance(node.datatype)
datatype.append_search_filters(val, node, tile_query,
request)
nested_query = Nested(path='tiles', query=tile_query)
if advanced_filter['op'] == 'or' and index != 0:
grouped_query = Bool()
grouped_queries.append(grouped_query)
grouped_query.must(nested_query)
for grouped_query in grouped_queries:
advanced_query.should(grouped_query)
search_query.must(advanced_query)
query.add_query(search_query)
if search_buffer != None:
search_buffer = search_buffer.geojson
return {'query': query, 'search_buffer': search_buffer}
def build_search_results_dsl(request):
term_filter = request.GET.get('termFilter', '')
spatial_filter = JSONDeserializer().deserialize(request.GET.get('mapFilter', '{}'))
export = request.GET.get('export', None)
page = 1 if request.GET.get('page') == '' else int(request.GET.get('page', 1))
temporal_filter = JSONDeserializer().deserialize(request.GET.get('temporalFilter', '{}'))
advanced_filters = JSONDeserializer().deserialize(request.GET.get('advanced', '[]'))
search_buffer = None
se = SearchEngineFactory().create()
if export != None:
limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE
else:
limit = settings.SEARCH_ITEMS_PER_PAGE
query = Query(se, start=limit*int(page-1), limit=limit)
nested_agg = NestedAgg(path='points', name='geo_aggs')
nested_agg.add_aggregation(GeoHashGridAgg(field='points.point', name='grid', precision=settings.HEX_BIN_PRECISION))
nested_agg.add_aggregation(GeoBoundsAgg(field='points.point', name='bounds'))
query.add_aggregation(nested_agg)
search_query = Bool()
permitted_nodegroups = get_permitted_nodegroups(request.user)
if term_filter != '':
for term in JSONDeserializer().deserialize(term_filter):
term_query = Bool()
if term['type'] == 'term' or term['type'] == 'string':
string_filter = Bool()
if term['type'] == 'term':
string_filter.must(Match(field='strings.string', query=term['value'], type='phrase'))
elif term['type'] == 'string':
string_filter.should(Match(field='strings.string', query=term['value'], type='phrase_prefix'))
string_filter.should(Match(field='strings.string.folded', query=term['value'], type='phrase_prefix'))
string_filter.filter(Terms(field='strings.nodegroup_id', terms=permitted_nodegroups))
nested_string_filter = Nested(path='strings', query=string_filter)
if term['inverted']:
search_query.must_not(nested_string_filter)
else:
search_query.must(nested_string_filter)
# need to set min_score because the query returns results with score 0 and those have to be removed, which I don't think it should be doing
query.min_score('0.01')
elif term['type'] == 'concept':
concept_ids = _get_child_concepts(term['value'])
conceptid_filter = Bool()
conceptid_filter.filter(Terms(field='domains.conceptid', terms=concept_ids))
conceptid_filter.filter(Terms(field='domains.nodegroup_id', terms=permitted_nodegroups))
nested_conceptid_filter = Nested(path='domains', query=conceptid_filter)
if term['inverted']:
search_query.must_not(nested_conceptid_filter)
else:
search_query.filter(nested_conceptid_filter)
if 'features' in spatial_filter:
if len(spatial_filter['features']) > 0:
feature_geom = spatial_filter['features'][0]['geometry']
feature_properties = spatial_filter['features'][0]['properties']
buffer = {'width':0,'unit':'ft'}
if 'buffer' in feature_properties:
buffer = feature_properties['buffer']
search_buffer = _buffer(feature_geom, buffer['width'], buffer['unit'])
feature_geom = JSONDeserializer().deserialize(search_buffer.json)
geoshape = GeoShape(field='geometries.geom.features.geometry', type=feature_geom['type'], coordinates=feature_geom['coordinates'] )
invert_spatial_search = False
if 'inverted' in feature_properties:
invert_spatial_search = feature_properties['inverted']
spatial_query = Bool()
if invert_spatial_search == True:
spatial_query.must_not(geoshape)
else:
spatial_query.filter(geoshape)
# get the nodegroup_ids that the user has permission to search
spatial_query.filter(Terms(field='geometries.nodegroup_id', terms=permitted_nodegroups))
search_query.filter(Nested(path='geometries', query=spatial_query))
if 'fromDate' in temporal_filter and 'toDate' in temporal_filter:
now = str(datetime.utcnow())
start_date = SortableDate(temporal_filter['fromDate'])
end_date = SortableDate(temporal_filter['toDate'])
date_nodeid = str(temporal_filter['dateNodeId']) if 'dateNodeId' in temporal_filter and temporal_filter['dateNodeId'] != '' else None
query_inverted = False if 'inverted' not in temporal_filter else temporal_filter['inverted']
temporal_query = Bool()
if query_inverted:
# inverted date searches need to use an OR clause and are generally more complicated to structure (can't use ES must_not)
# eg: less than START_DATE OR greater than END_DATE
inverted_date_query = Bool()
inverted_date_ranges_query = Bool()
if start_date.is_valid():
inverted_date_query.should(Range(field='dates.date', lt=start_date.as_float()))
inverted_date_ranges_query.should(Range(field='date_ranges.date_range', lt=start_date.as_float()))
if end_date.is_valid():
inverted_date_query.should(Range(field='dates.date', gt=end_date.as_float()))
inverted_date_ranges_query.should(Range(field='date_ranges.date_range', gt=end_date.as_float()))
date_query = Bool()
date_query.filter(inverted_date_query)
date_query.filter(Terms(field='dates.nodegroup_id', terms=permitted_nodegroups))
if date_nodeid:
date_query.filter(Term(field='dates.nodeid', term=date_nodeid))
else:
date_ranges_query = Bool()
date_ranges_query.filter(inverted_date_ranges_query)
date_ranges_query.filter(Terms(field='date_ranges.nodegroup_id', terms=permitted_nodegroups))
temporal_query.should(Nested(path='date_ranges', query=date_ranges_query))
temporal_query.should(Nested(path='dates', query=date_query))
else:
date_query = Bool()
date_query.filter(Range(field='dates.date', gte=start_date.as_float(), lte=end_date.as_float()))
date_query.filter(Terms(field='dates.nodegroup_id', terms=permitted_nodegroups))
if date_nodeid:
date_query.filter(Term(field='dates.nodeid', term=date_nodeid))
else:
date_ranges_query = Bool()
date_ranges_query.filter(Range(field='date_ranges.date_range', gte=start_date.as_float(), lte=end_date.as_float(), relation='intersects'))
date_ranges_query.filter(Terms(field='date_ranges.nodegroup_id', terms=permitted_nodegroups))
temporal_query.should(Nested(path='date_ranges', query=date_ranges_query))
temporal_query.should(Nested(path='dates', query=date_query))
search_query.filter(temporal_query)
#print search_query.dsl
datatype_factory = DataTypeFactory()
if len(advanced_filters) > 0:
advanced_query = Bool()
grouped_query = Bool()
grouped_queries = [grouped_query]
for index, advanced_filter in enumerate(advanced_filters):
tile_query = Bool()
for key, val in advanced_filter.iteritems():
if key != 'op':
node = models.Node.objects.get(pk=key)
if request.user.has_perm('read_nodegroup', node.nodegroup):
datatype = datatype_factory.get_instance(node.datatype)
datatype.append_search_filters(val, node, tile_query, request)
nested_query = Nested(path='tiles', query=tile_query)
if advanced_filter['op'] == 'or' and index != 0:
grouped_query = Bool()
grouped_queries.append(grouped_query)
grouped_query.must(nested_query)
for grouped_query in grouped_queries:
advanced_query.should(grouped_query)
search_query.must(advanced_query)
query.add_query(search_query)
if search_buffer != None:
search_buffer = search_buffer.geojson
return {'query': query, 'search_buffer':search_buffer}
def append_dsl(self, search_results_object, permitted_nodegroups,
include_provisional):
search_query = Bool()
querysting_params = self.request.GET.get(details['componentname'], '')
for term in JSONDeserializer().deserialize(querysting_params):
if term['type'] == 'term' or term['type'] == 'string':
string_filter = Bool()
if term['type'] == 'term':
string_filter.must(
Match(field='strings.string',
query=term['value'],
type='phrase'))
elif term['type'] == 'string':
string_filter.should(
Match(field='strings.string',
query=term['value'],
type='phrase_prefix'))
string_filter.should(
Match(field='strings.string.folded',
query=term['value'],
type='phrase_prefix'))
if include_provisional is False:
string_filter.must_not(
Match(field='strings.provisional',
query='true',
type='phrase'))
elif include_provisional == 'only provisional':
string_filter.must_not(
Match(field='strings.provisional',
query='false',
type='phrase'))
string_filter.filter(
Terms(field='strings.nodegroup_id',
terms=permitted_nodegroups))
nested_string_filter = Nested(path='strings',
query=string_filter)
if term['inverted']:
search_query.must_not(nested_string_filter)
else:
search_query.must(nested_string_filter)
# need to set min_score because the query returns results with score 0 and those have to be removed, which I don't think it should be doing
search_results_object['query'].min_score('0.01')
elif term['type'] == 'concept':
concept_ids = _get_child_concepts(term['value'])
conceptid_filter = Bool()
conceptid_filter.filter(
Terms(field='domains.conceptid', terms=concept_ids))
conceptid_filter.filter(
Terms(field='domains.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional is False:
conceptid_filter.must_not(
Match(field='domains.provisional',
query='true',
type='phrase'))
elif include_provisional == 'only provisional':
conceptid_filter.must_not(
Match(field='domains.provisional',
query='false',
type='phrase'))
nested_conceptid_filter = Nested(path='domains',
query=conceptid_filter)
if term['inverted']:
search_query.must_not(nested_conceptid_filter)
else:
search_query.filter(nested_conceptid_filter)
search_results_object['query'].add_query(search_query)
