def reverse_func(apps, schema_editor):
extensions = [os.path.join(settings.ONTOLOGY_PATH, x) for x in settings.ONTOLOGY_EXT]
management.call_command('load_ontology', source=os.path.join(settings.ONTOLOGY_PATH, settings.ONTOLOGY_BASE),
version=settings.ONTOLOGY_BASE_VERSION, ontology_name=settings.ONTOLOGY_BASE_NAME, id=settings.ONTOLOGY_BASE_ID, extensions=','.join(extensions), verbosity=0)
Node = apps.get_model("models", "Node")
Edge = apps.get_model("models", "Edge")
for node in Node.objects.all():
node.ontologyclass = str(node.ontologyclass).split('/')[-1]
node.save()
for edge in Edge.objects.all():
edge.ontologyproperty = str(edge.ontologyproperty).split('/')[-1]
edge.save()
# remove index for base Arches concept
se = SearchEngineFactory().create()
query = Query(se, start=0, limit=10000)
query.add_query(Term(field='conceptid', term='00000000-0000-0000-0000-000000000001'))
query.delete(index='concepts')
try:
DValueType = apps.get_model("models", "DValueType")
DValueType.objects.get(valuetype='identifier').delete()
except:
pass
def append_search_filters(self, value, node, query, request):
try:
if value['val'] != '':
term = True if value['val'] == 't' else False
query.must(Term(field='tiles.data.%s' % (str(node.pk)), term=term))
except KeyError, e:
pass
def delete_index(self):
se = SearchEngineFactory().create()
query = Query(se, start=0, limit=10000)
term = Term(field='id', term=self.id)
query.add_query(term)
query.delete(index='concept_labels')
se.delete_terms(self.id)
def reverse_func(apps, schema_editor):
Node = apps.get_model("models", "Node")
Edge = apps.get_model("models", "Edge")
for node in Node.objects.all():
node.ontologyclass = str(node.ontologyclass).split("/")[-1]
node.save()
for edge in Edge.objects.all():
edge.ontologyproperty = str(edge.ontologyproperty).split("/")[-1]
edge.save()
# remove index for base Arches concept
se = SearchEngineFactory().create()
query = Query(se, start=0, limit=10000)
query.add_query(
Term(field="conceptid", term="00000000-0000-0000-0000-000000000001"))
query.delete(index="concepts")
try:
DValueType = apps.get_model("models", "DValueType")
DValueType.objects.get(valuetype="identifier").delete()
except Exception:
pass
def delete_concept_values_index(concepts_to_delete):
se = SearchEngineFactory().create()
for concept in concepts_to_delete.itervalues():
query = Query(se, start=0, limit=10000)
term = Term(field='conceptid', term=concept.id)
query.add_query(term)
query.delete(index='strings', doc_type='concept')
def delete_concept_values_index(concepts_to_delete):
se = SearchEngineFactory().create()
for concept in concepts_to_delete.itervalues():
query = Query(se, start=0, limit=10000)
term = Term(field='conceptid', term=concept.id)
query.add_query(term)
query.delete(index='concept_labels')
for conceptvalue in concept.values:
se.delete_terms(conceptvalue.id)
def index_resources_by_type(resource_types, clear_index=True, index_name=None, batch_size=settings.BULK_IMPORT_BATCH_SIZE):
"""
Indexes all resources of a given type(s)
Arguments:
resource_types -- array of graph ids that represent resource types
Keyword Arguments:
clear_index -- set to True to remove all the resources of the types passed in from the index before the reindexing operation
index_name -- only applies to custom indexes and if given will try and just refresh the data in that index
batch_size -- the number of records to index as a group, the larger the number to more memory required
"""
status = ''
se = SearchEngineFactory().create()
datatype_factory = DataTypeFactory()
node_datatypes = {str(nodeid): datatype for nodeid, datatype in models.Node.objects.values_list('nodeid', 'datatype')}
status = ''
for resource_type in resource_types:
start = datetime.now()
resources = Resource.objects.filter(graph_id=str(resource_type))
graph_name = models.GraphModel.objects.get(graphid=str(resource_type)).name
print("Indexing resource type '{0}'".format(graph_name))
if index_name is None:
q = Query(se=se)
term = Term(field='graph_id', term=str(resource_type))
q.add_query(term)
if clear_index:
q.delete(index='resources', refresh=True)
with se.BulkIndexer(batch_size=batch_size, refresh=True) as doc_indexer:
with se.BulkIndexer(batch_size=batch_size, refresh=True) as term_indexer:
for resource in resources:
document, terms = resource.get_documents_to_index(fetchTiles=True, datatype_factory=datatype_factory, node_datatypes=node_datatypes)
doc_indexer.add(index='resources', id=document['resourceinstanceid'], data=document)
for term in terms:
term_indexer.add(index='terms', id=term['_id'], data=term['_source'])
result_summary = {'database': len(resources), 'indexed': se.count(index='resources', body=q.dsl)}
status = 'Passed' if result_summary['database'] == result_summary['indexed'] else 'Failed'
print("Status: {0}, Resource Type: {1}, In Database: {2}, Indexed: {3}, Took: {4} seconds".format(status, graph_name, result_summary['database'], result_summary['indexed'], (datetime.now()-start).seconds))
for index in settings.ELASTICSEARCH_CUSTOM_INDEXES:
es_index = import_class_from_string(index['module'])(index['name'])
es_index.bulk_index(resources=resources, resource_type=resource_type, graph_name=graph_name, clear_index=clear_index)
else:
es_index = get_index(index_name)
es_index.bulk_index(resources=resources, resource_type=resource_type, graph_name=graph_name, clear_index=clear_index)
return status
def bulk_index(self,
resources=None,
resource_type=None,
graph_name=None,
clear_index=True):
"""
Indexes a list of documents in bulk to Elastic Search
Arguments:
None
Keyword Arguments:
resources -- the list of resource instances to index
resource_type -- the type of resources being indexed
graph_name -- the name of the graph model that represents the resources being indexed
clear_index -- True(default) to remove all index records of type "resource_type" before indexing,
assumes that a field called "graph_id" exists on the indexed documents
Return: None
"""
start = datetime.now()
q = Query(se=self.se)
if clear_index:
term = Term(field="graph_id", term=str(resource_type))
q.add_query(term)
q.delete(index=self.index_name, refresh=True)
q = Query(se=self.se)
count_before = self.se.count(index=self.index_name, body=q.dsl)
result_summary = {"database": len(resources), "indexed": 0}
with self.se.BulkIndexer(batch_size=settings.BULK_IMPORT_BATCH_SIZE,
refresh=True) as indexer:
for resource in resources:
tiles = list(
models.TileModel.objects.filter(resourceinstance=resource))
document, doc_id = self.get_documents_to_index(resource, tiles)
if document is not None and id is not None:
indexer.add(index=self.index_name,
id=doc_id,
data=document)
result_summary["indexed"] = self.se.count(index=self.index_name,
body=q.dsl) - count_before
status = "Passed" if result_summary["database"] == result_summary[
"indexed"] else "Failed"
print("Custom Index - %s:" % self.index_name)
print(
" Status: {0}, Resource Type: {1}, In Database: {2}, Indexed: {3}, Took: {4} seconds"
.format(status, graph_name, result_summary["database"],
result_summary["indexed"],
(datetime.now() - start).seconds))
def get_resource_bounds(node):
query = Query(se, start=0, limit=0)
search_query = Bool()
query.add_query(search_query)
query.add_aggregation(
GeoBoundsAgg(field='points.point', name='bounds'))
query.add_query(
Term(field='graph_id', term=str(node.graph.graphid)))
results = query.search(index='resources')
bounds = results['aggregations']['bounds'][
'bounds'] if 'bounds' in results['aggregations'][
'bounds'] else None
return bounds
def get_resource_bounds(node):
query = Query(se, start=0, limit=0)
search_query = Bool()
query.add_query(search_query)
query.add_aggregation(
GeoBoundsAgg(field="points.point", name="bounds"))
query.add_query(
Term(field="graph_id", term=str(node.graph.graphid)))
results = query.search(index=RESOURCES_INDEX)
bounds = results["aggregations"]["bounds"][
"bounds"] if "bounds" in results["aggregations"][
"bounds"] else None
return bounds
def append_dsl(self, search_results_object, permitted_nodegroups,
include_provisional):
search_query = Bool()
querysting_params = self.request.GET.get(details['componentname'], '')
for resouceTypeFilter in JSONDeserializer().deserialize(
querysting_params):
term = Term(field='graph_id',
term=str(resouceTypeFilter['graphid']))
if resouceTypeFilter['inverted'] is True:
search_query.must_not(term)
else:
search_query.must(term)
search_results_object['query'].add_query(search_query)
def append_search_filters(self, value, node, query, request):
# Match the label in the same manner as a String datatype
try:
if value["val"] != "":
match_type = "phrase_prefix" if "~" in value["op"] else "phrase"
if "~" in value["op"]:
match_query = Match(
field="tiles.data.%s.url" % (str(node.pk)),
query=value["val"],
type=match_type,
)
if "=" in value["op"]:
match_query = Term(field="tiles.data.%s.url.keyword" % (str(node.pk)), term=value["val"])
if "!" in value["op"]:
query.must_not(match_query)
query.filter(Exists(field="tiles.data.%s" % (str(node.pk))))
else:
query.must(match_query)
except KeyError as e:
pass
def index_resources_by_type(resource_types,
clear_index=True,
batch_size=settings.BULK_IMPORT_BATCH_SIZE,
quiet=False):
"""
Indexes all resources of a given type(s)
Arguments:
resource_types -- array of graph ids that represent resource types
Keyword Arguments:
clear_index -- set to True to remove all the resources of the types passed in from the index before the reindexing operation
batch_size -- the number of records to index as a group, the larger the number to more memory required
quiet -- Silences the status bar output during certain operations, use in celery operations for example
"""
status = ""
datatype_factory = DataTypeFactory()
node_datatypes = {
str(nodeid): datatype
for nodeid, datatype in models.Node.objects.values_list(
"nodeid", "datatype")
}
if isinstance(resource_types, str):
resource_types = [resource_types]
for resource_type in resource_types:
start = datetime.now()
resources = Resource.objects.filter(graph_id=str(resource_type))
graph_name = models.GraphModel.objects.get(
graphid=str(resource_type)).name
print("Indexing resource type '{0}'".format(graph_name))
q = Query(se=se)
term = Term(field="graph_id", term=str(resource_type))
q.add_query(term)
if clear_index:
q.delete(index=RESOURCES_INDEX, refresh=True)
with se.BulkIndexer(batch_size=batch_size,
refresh=True) as doc_indexer:
with se.BulkIndexer(batch_size=batch_size,
refresh=True) as term_indexer:
if quiet is False:
bar = pyprind.ProgBar(
len(resources),
bar_char="█") if len(resources) > 1 else None
for resource in resources:
if quiet is False and bar is not None:
bar.update(item_id=resource)
document, terms = resource.get_documents_to_index(
fetchTiles=True,
datatype_factory=datatype_factory,
node_datatypes=node_datatypes)
doc_indexer.add(index=RESOURCES_INDEX,
id=document["resourceinstanceid"],
data=document)
for term in terms:
term_indexer.add(index=TERMS_INDEX,
id=term["_id"],
data=term["_source"])
result_summary = {
"database": len(resources),
"indexed": se.count(index=RESOURCES_INDEX, body=q.dsl)
}
status = "Passed" if result_summary["database"] == result_summary[
"indexed"] else "Failed"
print(
"Status: {0}, Resource Type: {1}, In Database: {2}, Indexed: {3}, Took: {4} seconds"
.format(status, graph_name, result_summary["database"],
result_summary["indexed"],
(datetime.now() - start).seconds))
return status
def build_search_results_dsl(request):
term_filter = request.GET.get('termFilter', '')
spatial_filter = JSONDeserializer().deserialize(
request.GET.get('mapFilter', '{}'))
include_provisional = get_provisional_type(request)
export = request.GET.get('export', None)
mobile_download = request.GET.get('mobiledownload', None)
page = 1 if request.GET.get('page') == '' else int(
request.GET.get('page', 1))
temporal_filter = JSONDeserializer().deserialize(
request.GET.get('temporalFilter', '{}'))
advanced_filters = JSONDeserializer().deserialize(
request.GET.get('advanced', '[]'))
search_buffer = None
se = SearchEngineFactory().create()
if export != None:
limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE
elif mobile_download != None:
limit = settings.MOBILE_DOWNLOAD_RESOURCE_LIMIT
else:
limit = settings.SEARCH_ITEMS_PER_PAGE
query = Query(se, start=limit * int(page - 1), limit=limit)
search_query = Bool()
nested_agg = NestedAgg(path='points', name='geo_aggs')
nested_agg_filter = FiltersAgg(name='inner')
if include_provisional == True:
nested_agg_filter.add_filter(
Terms(field='points.provisional', terms=['false', 'true']))
else:
provisional_resource_filter = Bool()
if include_provisional == False:
provisional_resource_filter.filter(
Terms(field='provisional', terms=['false', 'partial']))
nested_agg_filter.add_filter(
Terms(field='points.provisional', terms=['false']))
elif include_provisional == 'only provisional':
provisional_resource_filter.filter(
Terms(field='provisional', terms=['true', 'partial']))
nested_agg_filter.add_filter(
Terms(field='points.provisional', terms=['true']))
search_query.must(provisional_resource_filter)
nested_agg_filter.add_aggregation(
GeoHashGridAgg(field='points.point',
name='grid',
precision=settings.HEX_BIN_PRECISION))
nested_agg_filter.add_aggregation(
GeoBoundsAgg(field='points.point', name='bounds'))
nested_agg.add_aggregation(nested_agg_filter)
query.add_aggregation(nested_agg)
permitted_nodegroups = get_permitted_nodegroups(request.user)
if term_filter != '':
for term in JSONDeserializer().deserialize(term_filter):
term_query = Bool()
provisional_term_filter = Bool()
if term['type'] == 'term' or term['type'] == 'string':
string_filter = Bool()
if term['type'] == 'term':
string_filter.must(
Match(field='strings.string',
query=term['value'],
type='phrase'))
elif term['type'] == 'string':
string_filter.should(
Match(field='strings.string',
query=term['value'],
type='phrase_prefix'))
string_filter.should(
Match(field='strings.string.folded',
query=term['value'],
type='phrase_prefix'))
if include_provisional == False:
string_filter.must_not(
Match(field='strings.provisional',
query='true',
type='phrase'))
elif include_provisional == 'only provisional':
string_filter.must_not(
Match(field='strings.provisional',
query='false',
type='phrase'))
string_filter.filter(
Terms(field='strings.nodegroup_id',
terms=permitted_nodegroups))
nested_string_filter = Nested(path='strings',
query=string_filter)
if term['inverted']:
search_query.must_not(nested_string_filter)
else:
search_query.must(nested_string_filter)
# need to set min_score because the query returns results with score 0 and those have to be removed, which I don't think it should be doing
query.min_score('0.01')
elif term['type'] == 'concept':
concept_ids = _get_child_concepts(term['value'])
conceptid_filter = Bool()
conceptid_filter.filter(
Terms(field='domains.conceptid', terms=concept_ids))
conceptid_filter.filter(
Terms(field='domains.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional == False:
conceptid_filter.must_not(
Match(field='domains.provisional',
query='true',
type='phrase'))
elif include_provisional == 'only provisional':
conceptid_filter.must_not(
Match(field='domains.provisional',
query='false',
type='phrase'))
nested_conceptid_filter = Nested(path='domains',
query=conceptid_filter)
if term['inverted']:
search_query.must_not(nested_conceptid_filter)
else:
search_query.filter(nested_conceptid_filter)
if 'features' in spatial_filter:
if len(spatial_filter['features']) > 0:
feature_geom = spatial_filter['features'][0]['geometry']
feature_properties = {}
if 'properties' in spatial_filter['features'][0]:
feature_properties = spatial_filter['features'][0][
'properties']
buffer = {'width': 0, 'unit': 'ft'}
if 'buffer' in feature_properties:
buffer = feature_properties['buffer']
search_buffer = _buffer(feature_geom, buffer['width'],
buffer['unit'])
feature_geom = JSONDeserializer().deserialize(search_buffer.json)
geoshape = GeoShape(field='geometries.geom.features.geometry',
type=feature_geom['type'],
coordinates=feature_geom['coordinates'])
invert_spatial_search = False
if 'inverted' in feature_properties:
invert_spatial_search = feature_properties['inverted']
spatial_query = Bool()
if invert_spatial_search == True:
spatial_query.must_not(geoshape)
else:
spatial_query.filter(geoshape)
# get the nodegroup_ids that the user has permission to search
spatial_query.filter(
Terms(field='geometries.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional == False:
spatial_query.filter(
Terms(field='geometries.provisional', terms=['false']))
elif include_provisional == 'only provisional':
spatial_query.filter(
Terms(field='geometries.provisional', terms=['true']))
search_query.filter(Nested(path='geometries', query=spatial_query))
if 'fromDate' in temporal_filter and 'toDate' in temporal_filter:
now = str(datetime.utcnow())
start_date = ExtendedDateFormat(temporal_filter['fromDate'])
end_date = ExtendedDateFormat(temporal_filter['toDate'])
date_nodeid = str(
temporal_filter['dateNodeId']
) if 'dateNodeId' in temporal_filter and temporal_filter[
'dateNodeId'] != '' else None
query_inverted = False if 'inverted' not in temporal_filter else temporal_filter[
'inverted']
temporal_query = Bool()
if query_inverted:
# inverted date searches need to use an OR clause and are generally more complicated to structure (can't use ES must_not)
# eg: less than START_DATE OR greater than END_DATE
inverted_date_query = Bool()
inverted_date_ranges_query = Bool()
if start_date.is_valid():
inverted_date_query.should(
Range(field='dates.date', lt=start_date.lower))
inverted_date_ranges_query.should(
Range(field='date_ranges.date_range', lt=start_date.lower))
if end_date.is_valid():
inverted_date_query.should(
Range(field='dates.date', gt=end_date.upper))
inverted_date_ranges_query.should(
Range(field='date_ranges.date_range', gt=end_date.upper))
date_query = Bool()
date_query.filter(inverted_date_query)
date_query.filter(
Terms(field='dates.nodegroup_id', terms=permitted_nodegroups))
if include_provisional == False:
date_query.filter(
Terms(field='dates.provisional', terms=['false']))
elif include_provisional == 'only provisional':
date_query.filter(
Terms(field='dates.provisional', terms=['true']))
if date_nodeid:
date_query.filter(Term(field='dates.nodeid', term=date_nodeid))
else:
date_ranges_query = Bool()
date_ranges_query.filter(inverted_date_ranges_query)
date_ranges_query.filter(
Terms(field='date_ranges.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional == False:
date_ranges_query.filter(
Terms(field='date_ranges.provisional',
terms=['false']))
elif include_provisional == 'only provisional':
date_ranges_query.filter(
Terms(field='date_ranges.provisional', terms=['true']))
temporal_query.should(
Nested(path='date_ranges', query=date_ranges_query))
temporal_query.should(Nested(path='dates', query=date_query))
else:
date_query = Bool()
date_query.filter(
Range(field='dates.date',
gte=start_date.lower,
lte=end_date.upper))
date_query.filter(
Terms(field='dates.nodegroup_id', terms=permitted_nodegroups))
if include_provisional == False:
date_query.filter(
Terms(field='dates.provisional', terms=['false']))
elif include_provisional == 'only provisional':
date_query.filter(
Terms(field='dates.provisional', terms=['true']))
if date_nodeid:
date_query.filter(Term(field='dates.nodeid', term=date_nodeid))
else:
date_ranges_query = Bool()
date_ranges_query.filter(
Range(field='date_ranges.date_range',
gte=start_date.lower,
lte=end_date.upper,
relation='intersects'))
date_ranges_query.filter(
Terms(field='date_ranges.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional == False:
date_ranges_query.filter(
Terms(field='date_ranges.provisional',
terms=['false']))
if include_provisional == 'only provisional':
date_ranges_query.filter(
Terms(field='date_ranges.provisional', terms=['true']))
temporal_query.should(
Nested(path='date_ranges', query=date_ranges_query))
temporal_query.should(Nested(path='dates', query=date_query))
search_query.filter(temporal_query)
datatype_factory = DataTypeFactory()
if len(advanced_filters) > 0:
advanced_query = Bool()
grouped_query = Bool()
grouped_queries = [grouped_query]
for index, advanced_filter in enumerate(advanced_filters):
tile_query = Bool()
for key, val in advanced_filter.iteritems():
if key != 'op':
node = models.Node.objects.get(pk=key)
if request.user.has_perm('read_nodegroup', node.nodegroup):
datatype = datatype_factory.get_instance(node.datatype)
datatype.append_search_filters(val, node, tile_query,
request)
nested_query = Nested(path='tiles', query=tile_query)
if advanced_filter['op'] == 'or' and index != 0:
grouped_query = Bool()
grouped_queries.append(grouped_query)
grouped_query.must(nested_query)
for grouped_query in grouped_queries:
advanced_query.should(grouped_query)
search_query.must(advanced_query)
query.add_query(search_query)
if search_buffer != None:
search_buffer = search_buffer.geojson
return {'query': query, 'search_buffer': search_buffer}
def append_dsl(self, search_results_object, permitted_nodegroups,
include_provisional):
search_query = Bool()
querysting_params = self.request.GET.get(details["componentname"], "")
temporal_filter = JSONDeserializer().deserialize(querysting_params)
if "fromDate" in temporal_filter and "toDate" in temporal_filter:
# now = str(datetime.utcnow())
start_date = ExtendedDateFormat(temporal_filter["fromDate"])
end_date = ExtendedDateFormat(temporal_filter["toDate"])
date_nodeid = (str(temporal_filter["dateNodeId"])
if "dateNodeId" in temporal_filter
and temporal_filter["dateNodeId"] != "" else None)
query_inverted = False if "inverted" not in temporal_filter else temporal_filter[
"inverted"]
temporal_query = Bool()
if query_inverted:
# inverted date searches need to use an OR clause and are generally more complicated to structure (can't use ES must_not)
# eg: less than START_DATE OR greater than END_DATE
inverted_date_query = Bool()
inverted_date_ranges_query = Bool()
if start_date.is_valid():
inverted_date_query.should(
Range(field="dates.date", lt=start_date.lower))
inverted_date_ranges_query.should(
Range(field="date_ranges.date_range",
lt=start_date.lower))
if end_date.is_valid():
inverted_date_query.should(
Range(field="dates.date", gt=end_date.upper))
inverted_date_ranges_query.should(
Range(field="date_ranges.date_range",
gt=end_date.upper))
date_query = Bool()
date_query.filter(inverted_date_query)
date_query.filter(
Terms(field="dates.nodegroup_id",
terms=permitted_nodegroups))
if include_provisional is False:
date_query.filter(
Terms(field="dates.provisional", terms=["false"]))
elif include_provisional == "only provisional":
date_query.filter(
Terms(field="dates.provisional", terms=["true"]))
if date_nodeid:
date_query.filter(
Term(field="dates.nodeid", term=date_nodeid))
else:
date_ranges_query = Bool()
date_ranges_query.filter(inverted_date_ranges_query)
date_ranges_query.filter(
Terms(field="date_ranges.nodegroup_id",
terms=permitted_nodegroups))
if include_provisional is False:
date_ranges_query.filter(
Terms(field="date_ranges.provisional",
terms=["false"]))
elif include_provisional == "only provisional":
date_ranges_query.filter(
Terms(field="date_ranges.provisional",
terms=["true"]))
temporal_query.should(
Nested(path="date_ranges", query=date_ranges_query))
temporal_query.should(Nested(path="dates", query=date_query))
else:
date_query = Bool()
date_query.filter(
Range(field="dates.date",
gte=start_date.lower,
lte=end_date.upper))
date_query.filter(
Terms(field="dates.nodegroup_id",
terms=permitted_nodegroups))
if include_provisional is False:
date_query.filter(
Terms(field="dates.provisional", terms=["false"]))
elif include_provisional == "only provisional":
date_query.filter(
Terms(field="dates.provisional", terms=["true"]))
if date_nodeid:
date_query.filter(
Term(field="dates.nodeid", term=date_nodeid))
else:
date_ranges_query = Bool()
date_ranges_query.filter(
Range(field="date_ranges.date_range",
gte=start_date.lower,
lte=end_date.upper,
relation="intersects"))
date_ranges_query.filter(
Terms(field="date_ranges.nodegroup_id",
terms=permitted_nodegroups))
if include_provisional is False:
date_ranges_query.filter(
Terms(field="date_ranges.provisional",
terms=["false"]))
if include_provisional == "only provisional":
date_ranges_query.filter(
Terms(field="date_ranges.provisional",
terms=["true"]))
temporal_query.should(
Nested(path="date_ranges", query=date_ranges_query))
temporal_query.should(Nested(path="dates", query=date_query))
search_query.filter(temporal_query)
search_results_object["query"].add_query(search_query)
def append_dsl(self, search_results_object, permitted_nodegroups,
include_provisional):
search_query = Bool()
querysting_params = self.request.GET.get(details['componentname'], '')
temporal_filter = JSONDeserializer().deserialize(querysting_params)
if 'fromDate' in temporal_filter and 'toDate' in temporal_filter:
#now = str(datetime.utcnow())
start_date = ExtendedDateFormat(temporal_filter['fromDate'])
end_date = ExtendedDateFormat(temporal_filter['toDate'])
date_nodeid = str(
temporal_filter['dateNodeId']
) if 'dateNodeId' in temporal_filter and temporal_filter[
'dateNodeId'] != '' else None
query_inverted = False if 'inverted' not in temporal_filter else temporal_filter[
'inverted']
temporal_query = Bool()
if query_inverted:
# inverted date searches need to use an OR clause and are generally more complicated to structure (can't use ES must_not)
# eg: less than START_DATE OR greater than END_DATE
inverted_date_query = Bool()
inverted_date_ranges_query = Bool()
if start_date.is_valid():
inverted_date_query.should(
Range(field='dates.date', lt=start_date.lower))
inverted_date_ranges_query.should(
Range(field='date_ranges.date_range',
lt=start_date.lower))
if end_date.is_valid():
inverted_date_query.should(
Range(field='dates.date', gt=end_date.upper))
inverted_date_ranges_query.should(
Range(field='date_ranges.date_range',
gt=end_date.upper))
date_query = Bool()
date_query.filter(inverted_date_query)
date_query.filter(
Terms(field='dates.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional is False:
date_query.filter(
Terms(field='dates.provisional', terms=['false']))
elif include_provisional == 'only provisional':
date_query.filter(
Terms(field='dates.provisional', terms=['true']))
if date_nodeid:
date_query.filter(
Term(field='dates.nodeid', term=date_nodeid))
else:
date_ranges_query = Bool()
date_ranges_query.filter(inverted_date_ranges_query)
date_ranges_query.filter(
Terms(field='date_ranges.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional is False:
date_ranges_query.filter(
Terms(field='date_ranges.provisional',
terms=['false']))
elif include_provisional == 'only provisional':
date_ranges_query.filter(
Terms(field='date_ranges.provisional',
terms=['true']))
temporal_query.should(
Nested(path='date_ranges', query=date_ranges_query))
temporal_query.should(Nested(path='dates', query=date_query))
else:
date_query = Bool()
date_query.filter(
Range(field='dates.date',
gte=start_date.lower,
lte=end_date.upper))
date_query.filter(
Terms(field='dates.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional is False:
date_query.filter(
Terms(field='dates.provisional', terms=['false']))
elif include_provisional == 'only provisional':
date_query.filter(
Terms(field='dates.provisional', terms=['true']))
if date_nodeid:
date_query.filter(
Term(field='dates.nodeid', term=date_nodeid))
else:
date_ranges_query = Bool()
date_ranges_query.filter(
Range(field='date_ranges.date_range',
gte=start_date.lower,
lte=end_date.upper,
relation='intersects'))
date_ranges_query.filter(
Terms(field='date_ranges.nodegroup_id',
terms=permitted_nodegroups))
if include_provisional is False:
date_ranges_query.filter(
Terms(field='date_ranges.provisional',
terms=['false']))
if include_provisional == 'only provisional':
date_ranges_query.filter(
Terms(field='date_ranges.provisional',
terms=['true']))
temporal_query.should(
Nested(path='date_ranges', query=date_ranges_query))
temporal_query.should(Nested(path='dates', query=date_query))
search_query.filter(temporal_query)
search_results_object['query'].add_query(search_query)
def delete_index(self):
se = SearchEngineFactory().create()
query = Query(se, start=0, limit=10000)
term = Term(field='id', term=self.id)
query.add_query(term)
query.delete(index='strings', doc_type='concept')