def prepare_search_index(self, resource_type_id, create=False):
"""
Creates the settings and mappings in Elasticsearch to support resource search
"""
index_settings = super(Resource,
self).prepare_search_index(resource_type_id,
create=False)
index_settings['mappings'][resource_type_id]['properties'][
'date_groups'] = {
'properties': {
'conceptid': {
'type': 'string',
'index': 'not_analyzed'
}
}
}
if create:
se = SearchEngineFactory().create()
try:
se.create_index(index='entity', body=index_settings)
except:
index_settings = index_settings['mappings']
se.create_mapping(index='entity',
doc_type=resource_type_id,
body=index_settings)
def prepare_search_index(self, resource_type_id, create=False):
"""
Creates the settings and mappings in Elasticsearch to support resource search
"""
index_settings = super(Resource, self).prepare_search_index(resource_type_id, create=False)
index_settings['mappings'][resource_type_id]['properties']['date_groups'] = {
'properties' : {
'conceptid': {'type' : 'string', 'index' : 'not_analyzed'}
}
}
#index_settings['mappings'][resource_type_id]['properties']['measurement_groups'] = {
# 'properties' : {
# 'conceptid': {'type' : 'string', 'index' : 'not_analyzed'}
# }
#}
if create:
se = SearchEngineFactory().create()
try:
se.create_index(index='entity', body=index_settings)
except:
index_settings = index_settings['mappings']
se.create_mapping(index='entity', doc_type=resource_type_id, body=index_settings)
def index(self, scheme=None):
if self.category == 'label':
se = SearchEngineFactory().create()
data = JSONSerializer().serializeToPython(self)
if scheme == None:
scheme = self.get_scheme_id()
if scheme == None:
raise Exception('Index of label failed. Index type (scheme id) could not be derived from the label.')
se.create_mapping('concept_labels', scheme.id, fieldname='conceptid', fieldtype='string', fieldindex='not_analyzed')
se.index_data('concept_labels', scheme.id, data, 'id')
# don't create terms for entity type concepts
if not(scheme.id == '00000000-0000-0000-0000-000000000003' or scheme.id == '00000000-0000-0000-0000-000000000004'):
se.index_term(self.value, self.id, scheme.id, {'conceptid': self.conceptid})
def index(self, scheme=None):
if self.category == 'label':
se = SearchEngineFactory().create()
data = JSONSerializer().serializeToPython(self)
if scheme == None:
scheme = self.get_scheme_id()
if scheme == None:
raise Exception('Index of label failed. Index type (scheme id) could not be derived from the label.')
se.create_mapping('concept_labels', scheme.id, fieldname='conceptid', fieldtype='string', fieldindex='not_analyzed')
se.index_data('concept_labels', scheme.id, data, 'id')
# don't create terms for entity type concepts
if not(scheme.id == '00000000-0000-0000-0000-000000000003' or scheme.id == '00000000-0000-0000-0000-000000000004'):
se.index_term(self.value, self.id, scheme.id, {'conceptid': self.conceptid})
def index_concepts_for_search(self):
# see http://sqlblog.com/blogs/adam_machanic/archive/2006/07/12/swinging-from-tree-to-tree-using-ctes-part-1-adjacency-to-nested-sets.aspx
# Value of Lft for the root node is 1
# Value of Rgt for the root node is 2 * (Number of nodes)
# Value of Lft for any node is ((Number of nodes visited) * 2) - (Level of current node)
# Value of Rgt for any node is (Lft value) + ((Number of subnodes) * 2) + 1
sys.setrecursionlimit(3000)
se = SearchEngineFactory().create()
se.create_mapping('concept', 'all', 'conceptid', 'string', 'not_analyzed')
se.create_mapping('concept', 'all', 'labelid', 'string', 'not_analyzed')
def _findNarrowerConcept(conceptid, ret=None, limit=200000, level=1):
returnobj = {'subnodes': 0}
if ret == None: # the root node
labels = archesmodels.Values.objects.filter(conceptid = conceptid)
ret = {}
nodesvisited = len(ret) + 1
ret[conceptid] = {'labels': [], 'left': (nodesvisited*2)-level, 'right': 0}
for label in labels:
ret[conceptid]['labels'].append({'labelid': label.pk, 'label': label.value})
level = level + 1
conceptrealations = archesmodels.ConceptRelations.objects.filter(conceptidfrom = conceptid)
for relation in conceptrealations:
nodesvisited = len(ret) + 1
labels = archesmodels.Values.objects.filter(conceptid = relation.conceptidto)
ret[relation.conceptidto_id] = {'labels': [], 'left': (nodesvisited*2)-level, 'right': 0}
for label in labels:
ret[relation.conceptidto_id]['labels'].append({'labelid': label.pk, 'label': label.value})
returnobj = _findNarrowerConcept(relation.conceptidto_id, ret=ret, level=level+1)
subnodes = returnobj['subnodes']
if subnodes == 0: # meaning we're at a leaf node
ret[conceptid]['right'] = ret[conceptid]['left'] + 1
else:
ret[conceptid]['right'] = subnodes + 1
return {'all_concepts': ret, 'subnodes': ret[conceptid]['right']}
concepts = _findNarrowerConcept('00000000-0000-0000-0000-000000000003')
all_concepts = []
for key, concept in concepts['all_concepts'].iteritems():
all_concepts.append({'conceptid': key, 'labels': concept['labels'], 'left': concept['left'], 'right': concept['right']})
self.index(all_concepts, 'concept', 'all', 'conceptid')
def index_concepts_for_search(self):
# see http://sqlblog.com/blogs/adam_machanic/archive/2006/07/12/swinging-from-tree-to-tree-using-ctes-part-1-adjacency-to-nested-sets.aspx
# Value of Lft for the root node is 1
# Value of Rgt for the root node is 2 * (Number of nodes)
# Value of Lft for any node is ((Number of nodes visited) * 2) - (Level of current node)
# Value of Rgt for any node is (Lft value) + ((Number of subnodes) * 2) + 1
sys.setrecursionlimit(3000)
se = SearchEngineFactory().create()
se.create_mapping('concept', 'all', 'conceptid', 'string', 'not_analyzed')
se.create_mapping('concept', 'all', 'labelid', 'string', 'not_analyzed')
def _findNarrowerConcept(conceptid, ret=None, limit=200000, level=1):
returnobj = {'subnodes': 0}
if ret == None: # the root node
labels = archesmodels.Values.objects.filter(conceptid = conceptid)
ret = {}
nodesvisited = len(ret) + 1
ret[conceptid] = {'labels': [], 'left': (nodesvisited*2)-level, 'right': 0}
for label in labels:
ret[conceptid]['labels'].append({'labelid': label.pk, 'label': label.value})
level = level + 1
conceptrealations = archesmodels.ConceptRelations.objects.filter(conceptidfrom = conceptid)
for relation in conceptrealations:
nodesvisited = len(ret) + 1
labels = archesmodels.Values.objects.filter(conceptid = relation.conceptidto)
ret[relation.conceptidto_id] = {'labels': [], 'left': (nodesvisited*2)-level, 'right': 0}
for label in labels:
ret[relation.conceptidto_id]['labels'].append({'labelid': label.pk, 'label': label.value})
returnobj = _findNarrowerConcept(relation.conceptidto_id, ret=ret, level=level+1)
subnodes = returnobj['subnodes']
if subnodes == 0: # meaning we're at a leaf node
ret[conceptid]['right'] = ret[conceptid]['left'] + 1
else:
ret[conceptid]['right'] = subnodes + 1
return {'all_concepts': ret, 'subnodes': ret[conceptid]['right']}
concepts = _findNarrowerConcept('00000000-0000-0000-0000-000000000003')
all_concepts = []
for key, concept in concepts['all_concepts'].iteritems():
all_concepts.append({'conceptid': key, 'labels': concept['labels'], 'left': concept['left'], 'right': concept['right']})
self.index(all_concepts, 'concept', 'all', 'conceptid')
def index(self, scheme=None):
if self.category == 'label':
se = SearchEngineFactory().create()
data = JSONSerializer().serializeToPython(self)
if scheme == None:
scheme = self.get_scheme_id()
if scheme == None:
raise Exception('Index of label failed. Index type (scheme id) could not be derived from the label.')
se.create_mapping('concept_labels', scheme.id, fieldname='conceptid', fieldtype='string', fieldindex='not_analyzed')
se.index_data('concept_labels', scheme.id, data, 'id')
#Looks up whether the label is actually a dropdown label or an entity label and, if so, excludes them from the term search index.
entity_or_dropdown= archesmodels.ConceptRelations.objects.filter(Q(relationtype ='hasCollection') | Q(relationtype ='hasEntity'),conceptidto = scheme.id)
is_entity_or_dropdown = False if entity_or_dropdown.count() == 0 else True
# don't create terms for entity type concepts
if not(scheme.id == '00000000-0000-0000-0000-000000000003' or scheme.id == '00000000-0000-0000-0000-000000000004') and is_entity_or_dropdown ==False:
se.index_term(self.value, self.id, scheme.id, {'conceptid': self.conceptid})
def index(self, scheme=None):
if self.category == 'label':
se = SearchEngineFactory().create()
data = JSONSerializer().serializeToPython(self)
if scheme == None:
scheme = self.get_scheme_id()
if scheme == None:
raise Exception('Index of label failed. Index type (scheme id) could not be derived from the label.')
se.create_mapping('concept_labels', scheme.id, fieldname='conceptid', fieldtype='string', fieldindex='not_analyzed')
se.index_data('concept_labels', scheme.id, data, 'id')
#Looks up whether the label is actually a dropdown label or an entity label and, if so, excludes them from the term search index.
entity_or_dropdown= archesmodels.ConceptRelations.objects.filter(Q(relationtype ='hasCollection') | Q(relationtype ='hasEntity'),conceptidto = scheme.id)
is_entity_or_dropdown = False if entity_or_dropdown.count() == 0 else True
# don't create terms for entity type concepts
if not(scheme.id == '00000000-0000-0000-0000-000000000003' or scheme.id == '00000000-0000-0000-0000-000000000004') and is_entity_or_dropdown ==False:
se.index_term(self.value, self.id, scheme.id, {'conceptid': self.conceptid})
def forwards_func(apps, schema_editor):
se = SearchEngineFactory().create()
body = {
'_doc': {
'properties': {
'ids': {
'type': 'nested',
'properties': {
'id': {'type': 'keyword'},
'nodegroup_id': {'type': 'keyword'},
'provisional': {'type': 'boolean'}
}
}
}
}
}
se.create_mapping(index='resources', body=body)
def prepare_search_index(self, resource_type_id, create=False):
"""
Creates the settings and mappings in Elasticsearch to support resource search
"""
index_settings = {
'settings':{
'analysis': {
'analyzer': {
'folding': {
'tokenizer': 'standard',
'filter': [ 'lowercase', 'asciifolding' ]
}
}
}
},
'mappings': {
resource_type_id : {
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'primaryname': {'type' : 'string', 'index' : 'not_analyzed'},
'child_entities' : {
'type' : 'nested',
'index' : 'analyzed',
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {
'type' : 'string',
'index' : 'analyzed',
'fields' : {
'raw' : { 'type' : 'string', 'index' : 'not_analyzed'},
'folded': { 'type': 'string', 'analyzer': 'folding'}
}
}
}
},
'domains' : {
'type' : 'nested',
'index' : 'analyzed',
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {
'type' : 'string',
'index' : 'analyzed',
'fields' : {
'raw' : { 'type' : 'string', 'index' : 'not_analyzed'}
}
},
'conceptid' : {'type' : 'string', 'index' : 'not_analyzed'},
}
},
'geometries' : {
'type' : 'nested',
'index' : 'analyzed',
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {
"type": "geo_shape"
}
}
},
'dates' : {
'type' : 'nested',
'index' : 'analyzed',
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {
"type" : "date"
}
}
}
}
}
}
}
if create:
se = SearchEngineFactory().create()
try:
se.create_index(index='entity', body=index_settings)
except:
index_settings = index_settings['mappings']
se.create_mapping(index='entity', doc_type=resource_type_id, body=index_settings)
return index_settings
def prepare_search_index(resource_model_id, create=False):
"""
Creates the settings and mappings in Elasticsearch to support resource search
"""
index_settings = {
'settings': {
'analysis': {
'analyzer': {
'folding': {
'tokenizer': 'standard',
'filter': ['lowercase', 'asciifolding']
}
}
}
},
'mappings': {
resource_model_id: {
'properties': {
'graphid': {
'type': 'keyword'
},
'resourceinstanceid': {
'type': 'keyword'
},
'root_ontology_class': {
'type': 'keyword'
},
'displayname': {
'type': 'keyword'
},
'displaydescription': {
'type': 'keyword'
},
'map_popup': {
'type': 'keyword'
},
'provisional': {
'type': 'keyword'
},
'tiles': {
'type': 'nested',
'properties': {
"tiles": {
'enabled': False
},
'tileid': {
'type': 'keyword'
},
'nodegroup_id': {
'type': 'keyword'
},
'parenttile_id': {
'type': 'keyword'
},
'resourceinstanceid_id': {
'type': 'keyword'
}
}
},
'strings': {
'type': 'nested',
'properties': {
'string': {
'type': 'text',
'index': 'analyzed',
'fields': {
'raw': {
'type': 'keyword'
},
'folded': {
'type': 'text',
'analyzer': 'folding'
}
}
},
'nodegroup_id': {
'type': 'keyword'
},
'provisional': {
'type': 'keyword'
}
}
},
'domains': {
'type': 'nested',
'properties': {
'value': {
'type': 'text',
'index': 'analyzed',
'fields': {
'raw': {
'type': 'keyword'
}
}
},
'conceptid': {
'type': 'keyword'
},
'valueid': {
'type': 'keyword'
},
'nodegroup_id': {
'type': 'keyword'
},
'provisional': {
'type': 'keyword'
}
}
},
'geometries': {
'type': 'nested',
'properties': {
'geom': {
'properties': {
'features': {
'properties': {
'geometry': {
'type': 'geo_shape'
},
'id': {
'type': 'keyword'
},
'type': {
'type': 'keyword'
},
'properties': {
'enabled': False
}
}
},
'type': {
'type': 'keyword'
}
}
},
'nodegroup_id': {
'type': 'keyword'
},
'provisional': {
'type': 'keyword'
}
}
},
'points': {
'type': 'nested',
'properties': {
'point': {
'type': 'geo_point'
},
'nodegroup_id': {
'type': 'keyword'
},
'provisional': {
'type': 'keyword'
}
}
},
'dates': {
'type': 'nested',
'properties': {
'date': {
'type': 'float'
},
'nodegroup_id': {
'type': 'keyword'
},
'nodeid': {
'type': 'keyword'
},
'provisional': {
'type': 'keyword'
}
}
},
'numbers': {
'type': 'nested',
'properties': {
'number': {
'type': 'double'
},
'nodegroup_id': {
'type': 'keyword'
},
'provisional': {
'type': 'keyword'
}
}
},
'date_ranges': {
'type': 'nested',
'properties': {
'date_range': {
'type': 'float_range'
},
'nodegroup_id': {
'type': 'keyword'
},
'provisional': {
'type': 'keyword'
}
}
}
}
}
}
}
if create:
se = SearchEngineFactory().create()
try:
se.create_index(index='resource', body=index_settings)
except:
index_settings = index_settings['mappings']
se.create_mapping(index='resource',
doc_type=resource_model_id,
body=index_settings)
return index_settings
def prepare_search_index(resource_model_id, create=False):
"""
Creates the settings and mappings in Elasticsearch to support resource search
"""
index_settings = {
'settings':{
'analysis': {
'analyzer': {
'folding': {
'tokenizer': 'standard',
'filter': [ 'lowercase', 'asciifolding' ]
}
}
}
},
'mappings': {
resource_model_id : {
'properties' : {
'graphid': {'type' : 'string', 'index' : 'not_analyzed'},
'resourceinstanceid': {'type' : 'string', 'index' : 'not_analyzed'},
'primaryname': {'type' : 'string', 'index' : 'not_analyzed'},
'tiles' : {
'type' : 'nested',
'properties' : {
"tiles": {'enabled': False},
'tileid' : {'type' : 'string', 'index' : 'not_analyzed'},
'nodegroup_id' : {'type' : 'string', 'index' : 'not_analyzed'},
'parenttile_id' : {'type' : 'string', 'index' : 'not_analyzed'},
'resourceinstanceid_id' : {'type' : 'string', 'index' : 'not_analyzed'}
}
},
'strings' : {
'type' : 'string',
'index' : 'analyzed',
'fields' : {
'raw' : { 'type' : 'string', 'index' : 'not_analyzed'},
'folded': { 'type': 'string', 'analyzer': 'folding'}
}
},
'domains' : {
'properties' : {
'value' : {
'type' : 'string',
'index' : 'analyzed',
'fields' : {
'raw' : { 'type' : 'string', 'index' : 'not_analyzed'}
}
},
'conceptid' : {'type' : 'string', 'index' : 'not_analyzed'},
'valueid' : {'type' : 'string', 'index' : 'not_analyzed'},
}
},
'geometries' : {
"properties": {
"features": {
"properties": {
"geometry": {"type": "geo_shape"},
"id": { 'type' : 'string', 'index' : 'not_analyzed'},
"type": { 'type' : 'string', 'index' : 'not_analyzed'},
"properties": {
"enabled": False
}
}
},
"type": { 'type' : 'string', 'index' : 'not_analyzed'}
}
},
'dates' : {
"type" : "date"
},
'numbers' : {
"type" : "double"
}
}
}
}
}
if create:
se = SearchEngineFactory().create()
try:
se.create_index(index='resource', body=index_settings)
except:
index_settings = index_settings['mappings']
se.create_mapping(index='resource', doc_type=resource_model_id, body=index_settings)
return index_settings
def prepare_search_index(resource_model_id, create=False):
"""
Creates the settings and mappings in Elasticsearch to support resource search
"""
index_settings = {
'settings': {
'analysis': {
'analyzer': {
'folding': {
'tokenizer': 'standard',
'filter': [ 'lowercase', 'asciifolding' ]
}
}
}
},
'mappings': {
resource_model_id : {
'properties' : {
'graphid': {'type': 'keyword'},
'resourceinstanceid': {'type': 'keyword'},
'root_ontology_class': {'type':'keyword'},
'displayname': {'type': 'keyword'},
'displaydescription': {'type': 'keyword'},
'map_popup': {'type': 'keyword'},
'provisional': {'type': 'keyword'},
'tiles' : {
'type' : 'nested',
'properties' : {
"tiles": {'enabled': False},
'tileid' : {'type': 'keyword'},
'nodegroup_id' : {'type': 'keyword'},
'parenttile_id' : {'type': 'keyword'},
'resourceinstanceid_id' : {'type': 'keyword'}
}
},
'strings' : {
'type' : 'nested',
'properties': {
'string': {
'type' : 'text',
'index' : 'analyzed',
'fields' : {
'raw' : {'type': 'keyword'},
'folded': { 'type': 'text', 'analyzer': 'folding'}
}
},
'nodegroup_id' : {'type': 'keyword'},
'provisional': {'type': 'keyword'}
}
},
'domains' : {
'type' : 'nested',
'properties' : {
'value' : {
'type' : 'text',
'index' : 'analyzed',
'fields' : {
'raw' : {'type': 'keyword'}
}
},
'conceptid' : {'type': 'keyword'},
'valueid' : {'type': 'keyword'},
'nodegroup_id' : {'type': 'keyword'},
'provisional': {'type': 'keyword'}
}
},
'geometries' : {
'type' : 'nested',
'properties': {
'geom': {
'properties': {
'features': {
'properties': {
'geometry': {'type': 'geo_shape'},
'id': {'type': 'keyword'},
'type': {'type': 'keyword'},
'properties': {
'enabled': False
}
}
},
'type': {'type': 'keyword'}
}
},
'nodegroup_id' : {'type': 'keyword'},
'provisional': {'type': 'keyword'}
}
},
'points': {
'type' : 'nested',
'properties' : {
'point' : {'type': 'geo_point'},
'nodegroup_id' : {'type': 'keyword'},
'provisional': {'type': 'keyword'}
}
},
'dates' : {
'type' : 'nested',
'properties' : {
'date' : {'type': 'float'},
'nodegroup_id' : {'type': 'keyword'},
'nodeid' : {'type': 'keyword'},
'provisional': {'type': 'keyword'}
}
},
'numbers' : {
'type' : 'nested',
'properties' : {
'number' : {'type': 'double'},
'nodegroup_id' : {'type': 'keyword'},
'provisional': {'type': 'keyword'}
}
},
'date_ranges': {
'type' : 'nested',
'properties' : {
'date_range' : {'type': 'float_range'},
'nodegroup_id' : {'type': 'keyword'},
'provisional': {'type': 'keyword'}
}
}
}
}
}
}
if create:
se = SearchEngineFactory().create()
try:
se.create_index(index='resource', body=index_settings)
except:
index_settings = index_settings['mappings']
se.create_mapping(index='resource', doc_type=resource_model_id, body=index_settings)
return index_settings
def prepare_search_index(self, resource_type_id, create=False):
"""
Creates the settings and mappings in Elasticsearch to support resource search
"""
index_settings = {
'settings':{
'analysis': {
'analyzer': {
'folding': {
'tokenizer': 'standard',
'filter': [ 'lowercase', 'asciifolding' ]
}
}
}
},
'mappings': {
resource_type_id : {
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'primaryname': {'type' : 'string', 'index' : 'not_analyzed'},
'child_entities' : {
'type' : 'nested',
'index' : 'analyzed',
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {
'type' : 'string',
'index' : 'analyzed',
'fields' : {
'raw' : { 'type' : 'string', 'index' : 'not_analyzed'},
'folded': { 'type': 'string', 'analyzer': 'folding'}
}
}
}
},
'domains' : {
'type' : 'nested',
'index' : 'analyzed',
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {
'type' : 'string',
'index' : 'analyzed',
'fields' : {
'raw' : { 'type' : 'string', 'index' : 'not_analyzed'}
}
},
'conceptid' : {'type' : 'string', 'index' : 'not_analyzed'},
}
},
'geometries' : {
'type' : 'nested',
'index' : 'analyzed',
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {
"type": "geo_shape"
}
}
},
'dates' : {
'type' : 'nested',
'index' : 'analyzed',
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {
"type" : "date"
}
}
},
'extendeddates' : {
'type' : 'nested',
'index' : 'analyzed',
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {
'type' : 'integer'
}
}
},
'extendeddategroups' : {
'properties' : {
'conceptid': {'type' : 'string', 'index' : 'not_analyzed'},
'value': {'type' : 'integer', 'index' : 'not_analyzed'}
}
}
}
}
}
}
if create:
se = SearchEngineFactory().create()
try:
se.create_index(index='entity', body=index_settings)
except:
index_settings = index_settings['mappings']
se.create_mapping(index='entity', doc_type=resource_type_id, body=index_settings)
return index_settings
def base_prepare_search_index(self, resource_type_id, create=False):
"""
Creates the settings and mappings in Elasticsearch to support resource search
"""
index_settings = {
'settings':{
'analysis': {
"filter": {
"desc_ngram": {
"type": "ngram",
"min_gram": 2,
"max_gram": 30
}
},
'analyzer': {
'folding': {
'tokenizer': 'standard',
'filter': [ 'lowercase', 'asciifolding']
},
'ducet_sort': {
'tokenizer': 'keyword',
'filter': [ 'icu_collation']
},
"index_ngram": {
"type": "custom",
"tokenizer": "keyword",
"filter": [ "desc_ngram", "lowercase" ]
},
"search_ngram": {
"type": "custom",
"tokenizer": "keyword",
"filter": ["standard", "lowercase"]
}
}
}
},
'mappings': {
resource_type_id : {
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'primaryname': {'type' : 'string', 'index' : 'not_analyzed'},
'child_entities' : {
'type' : 'nested',
'index' : 'analyzed',
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'analyzer': 'ducet_sort'},
'value' : {
'type' : 'string',
"index_analyzer": "index_ngram",
"search_analyzer": "search_ngram",
'fields' : {
'raw' : { 'type' : 'string', 'index' : 'not_analyzed'},
'folded': { 'type': 'string', 'analyzer': 'folding'}
}
}
}
},
'domains' : {
'type' : 'nested',
'index' : 'analyzed',
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'analyzer': 'ducet_sort'},
'value' : {
'type' : 'string',
'index' : 'analyzed',
'fields' : {
'raw' : { 'type' : 'string', 'index' : 'not_analyzed'}
}
},
'conceptid' : {'type' : 'string', 'index' : 'not_analyzed'},
}
},
'geometries' : {
'type' : 'nested',
'index' : 'analyzed',
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {
"type": "geo_shape"
}
}
},
'dates' : {
'type' : 'nested',
'index' : 'analyzed',
'properties' : {
'entityid' : {'type' : 'string', 'index' : 'not_analyzed'},
'parentid' : {'type' : 'string', 'index' : 'not_analyzed'},
'property' : {'type' : 'string', 'index' : 'not_analyzed'},
'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'},
'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'},
'label' : {'type' : 'string', 'index' : 'not_analyzed'},
'value' : {
"type" : "date"
}
}
}
}
}
}
}
if create:
se = SearchEngineFactory().create()
try:
se.create_index(index='entity', body=index_settings)
except:
index_settings = index_settings['mappings']
se.create_mapping(index='entity', doc_type=resource_type_id, body=index_settings)
return index_settings
