def test_bulk_add_documents(self): """ Test adding documents to Elasticsearch in bulk """ se = SearchEngineFactory().create() se.create_index(index="test") documents = [] count_before = se.count(index="test") for i in range(10): doc = { "id": i, "type": "prefLabel", "value": "test pref label", } documents.append( se.create_bulk_item(op_type="index", index="test", id=doc["id"], data=doc)) ret = se.bulk_index(documents, refresh=True) count_after = se.count(index="test") self.assertEqual(count_after - count_before, 10)
def test_bulk_add_documents(self): """ Test adding documents to Elasticsearch in bulk """ se = SearchEngineFactory().create() se.create_index(index='test') documents = [] count_before = se.count(index='test') for i in range(10): doc = { 'id': i, 'type': 'prefLabel', 'value': 'test pref label', } documents.append( se.create_bulk_item(op_type='index', index='test', id=doc['id'], data=doc)) ret = se.bulk_index(documents, refresh=True) count_after = se.count(index='test') self.assertEqual(count_after - count_before, 10)
def setUpClass(cls): se = SearchEngineFactory().create() se.delete_index(index='concept_labels') se.delete_index(index='term') se.create_index(index='concept_labels') se.create_index(index='term') management.call_command('packages', operation='import_json', source='tests/fixtures/resource_graphs/archesv4_resource.json')
def prepare_concepts_index(create=False): """ Creates the settings and mappings in Elasticsearch to support term search """ index_settings = { "settings": {"analysis": {"analyzer": {"folding": {"tokenizer": "standard", "filter": ["lowercase", "asciifolding"]}}}}, "mappings": { "_doc": { "properties": { "top_concept": {"type": "keyword"}, "conceptid": {"type": "keyword"}, "language": {"type": "keyword"}, "id": {"type": "keyword"}, "category": {"type": "keyword"}, "provisional": {"type": "boolean"}, "type": {"type": "keyword"}, "value": { "analyzer": "standard", "type": "text", "fields": {"raw": {"type": "keyword"}, "folded": {"analyzer": "folding", "type": "text"}}, }, } } }, } if create: se = SearchEngineFactory().create() se.create_index(index="concepts", body=index_settings) return index_settings
def prepare_resource_relations_index(self, create=False): """ Creates the settings and mappings in Elasticsearch to support related resources """ index_settings = { 'mappings':{ 'all': { 'properties': { 'resourcexid': {'type': 'long'}, 'notes': { 'type': 'string'}, 'relationshiptype': {'type': 'string', 'index' : 'not_analyzed'}, 'entityid2': {'type': 'string', 'index' : 'not_analyzed'}, 'entityid1': {'type': 'string', 'index' : 'not_analyzed'} } } } } if create: se = SearchEngineFactory().create() se.create_index(index='resource_relations', body=index_settings, ignore=400) return index_settings
def prepare_search_index(self, resource_type_id, create=False): """ Creates the settings and mappings in Elasticsearch to support resource search """ index_settings = super(Resource, self).prepare_search_index(resource_type_id, create=False) index_settings['mappings'][resource_type_id]['properties']['date_groups'] = { 'properties' : { 'conceptid': {'type' : 'string', 'index' : 'not_analyzed'} } } #index_settings['mappings'][resource_type_id]['properties']['measurement_groups'] = { # 'properties' : { # 'conceptid': {'type' : 'string', 'index' : 'not_analyzed'} # } #} if create: se = SearchEngineFactory().create() try: se.create_index(index='entity', body=index_settings) except: index_settings = index_settings['mappings'] se.create_mapping(index='entity', doc_type=resource_type_id, body=index_settings)
def prepare_resource_relations_index(create=False): """ Creates the settings and mappings in Elasticsearch to support related resources """ index_settings = { "mappings": { "_doc": { "properties": { "resourcexid": {"type": "keyword"}, "notes": {"type": "text"}, "relationshiptype": {"type": "keyword"}, "resourceinstanceidfrom": {"type": "keyword"}, "resourceinstanceidto": {"type": "keyword"}, "created": {"type": "keyword"}, "modified": {"type": "keyword"}, } } } } if create: se = SearchEngineFactory().create() se.create_index(index="resource_relations", body=index_settings) return index_settings
def prepare_resource_relations_index(create=False): """ Creates the settings and mappings in Elasticsearch to support related resources """ index_settings = { "mappings": { "_doc": { "properties": { "resourcexid": {"type": "keyword"}, "notes": {"type": "text"}, "relationshiptype": {"type": "keyword"}, "inverserelationshiptype": {"type": "keyword"}, "resourceinstanceidfrom": {"type": "keyword"}, "resourceinstancefrom_graphid": {"type": "keyword"}, "resourceinstanceidto": {"type": "keyword"}, "resourceinstanceto_graphid": {"type": "keyword"}, "created": {"type": "keyword"}, "modified": {"type": "keyword"}, "datestarted": {"type": "date"}, "dateended": {"type": "date"}, "tileid": {"type": "keyword"}, "nodeid": {"type": "keyword"}, } } } } if create: se = SearchEngineFactory().create() se.create_index(index=RESOURCE_RELATIONS_INDEX, body=index_settings) return index_settings
def prepare_terms_index(create=False): """ Creates the settings and mappings in Elasticsearch to support term search """ index_settings = { "settings": {"analysis": {"analyzer": {"folding": {"tokenizer": "standard", "filter": ["lowercase", "asciifolding"]}}}}, "mappings": { "_doc": { "properties": { "nodegroupid": {"type": "keyword"}, "tileid": {"type": "keyword"}, "nodeid": {"type": "keyword"}, "resourceinstanceid": {"type": "keyword"}, "provisional": {"type": "boolean"}, "value": { "analyzer": "standard", "type": "text", "fields": {"raw": {"type": "keyword"}, "folded": {"analyzer": "folding", "type": "text"}}, }, } } }, } if create: se = SearchEngineFactory().create() se.create_index(index=TERMS_INDEX, body=index_settings) return index_settings
def prepare_resource_relations_index(create=False): """ Creates the settings and mappings in Elasticsearch to support related resources """ index_settings = { 'mappings': { 'all': { 'properties': { 'resourcexid': {'type': 'keyword'}, 'notes': {'type': 'text'}, 'relationshiptype': {'type': 'keyword'}, 'resourceinstanceidfrom': {'type': 'keyword'}, 'resourceinstanceidto': {'type': 'keyword'}, 'created': {'type': 'keyword'}, 'modified': {'type': 'keyword'} } } } } if create: se = SearchEngineFactory().create() se.create_index(index='resource_relations', body=index_settings, ignore=400) return index_settings
def prepare_resource_relations_index(create=False): """ Creates the settings and mappings in Elasticsearch to support related resources """ index_settings = { 'mappings': { 'all': { 'properties': { 'resourcexid': {'type': 'keyword'}, 'notes': {'type': 'text'}, 'relationshiptype': {'type': 'keyword'}, 'resourceinstanceidfrom': {'type': 'keyword'}, 'resourceinstanceidto': {'type': 'keyword'}, 'created': {'type': 'keyword'}, 'modified': {'type': 'keyword'} } } } } if create: se = SearchEngineFactory().create() se.create_index(index='resource_relations', body=index_settings, ignore=400) return index_settings
def reverse_func(apps, schema_editor): se = SearchEngineFactory().create() prefix = settings.ELASTICSEARCH_PREFIX if (se.es.indices.exists(index="%s_resource_relations" % prefix)): index_settings = prepare_resource_relations_index(create=False) index_settings['mappings']['all'] = index_settings['mappings'][ '_doc'] index_settings['mappings'].pop('_doc', None) se.create_index(index='resource_relations_temp', body=index_settings) doc = { "source": { "index": "%s_resource_relations" % prefix, "type": "_doc" }, "dest": { "index": "%s_resource_relations_temp" % prefix, "type": "all" } } se.es.reindex(body=doc, refresh=True, wait_for_completion=True) se.delete_index(index='resource_relations') se.create_index(index='resource_relations', body=index_settings) doc = { "source": { "index": "%s_resource_relations_temp" % prefix }, "dest": { "index": "%s_resource_relations" % prefix, "type": "all" } } se.es.reindex(body=doc, refresh=True, wait_for_completion=True)
def prepare_search_index(self, resource_type_id, create=False): """ Creates the settings and mappings in Elasticsearch to support resource search """ index_settings = super(Resource, self).prepare_search_index(resource_type_id, create=False) index_settings['mappings'][resource_type_id]['properties'][ 'date_groups'] = { 'properties': { 'conceptid': { 'type': 'string', 'index': 'not_analyzed' } } } if create: se = SearchEngineFactory().create() try: se.create_index(index='entity', body=index_settings) except: index_settings = index_settings['mappings'] se.create_mapping(index='entity', doc_type=resource_type_id, body=index_settings)
def setUpClass(cls): se = SearchEngineFactory().create() se.delete_index(index="concept_labels") se.delete_index(index="term") se.create_index(index="concept_labels") se.create_index(index="term") management.call_command( "packages", operation="import_json", source="tests/fixtures/resource_graphs/archesv4_resource.json" )
def test_bulk_indexer(self): se = SearchEngineFactory().create() se.create_index(index="bulk") with se.BulkIndexer(batch_size=500, refresh=True) as bulk_indexer: for i in range(1001): doc = {"id": i, "type": "prefLabel", "value": "test pref label"} bulk_indexer.add(index="bulk", id=doc["id"], data=doc) count_after = se.count(index="bulk") self.assertEqual(count_after, 1001)
def test_bulk_indexer(self): se = SearchEngineFactory().create() se.create_index(index='bulk') with se.BulkIndexer(batch_size=500, refresh=True) as bulk_indexer: for i in range(1001): doc = { 'id': i, 'type': 'prefLabel', 'value': 'test pref label', } bulk_indexer.add(index='bulk', id=doc['id'], data=doc) count_after = se.count(index='bulk') self.assertEqual(count_after, 1001)
def prepare_concepts_index(create=False): """ Creates the settings and mappings in Elasticsearch to support term search """ index_settings = { 'settings': { 'analysis': { 'analyzer': { 'folding': { 'tokenizer': 'standard', 'filter': ['lowercase', 'asciifolding'] } } } }, 'mappings': { '_doc': { 'properties': { 'top_concept': {'type': 'keyword'}, 'conceptid': {'type': 'keyword'}, 'language': {'type': 'keyword'}, 'id': {'type': 'keyword'}, 'category': {'type': 'keyword'}, 'provisional': {'type': 'boolean'}, 'type': {'type': 'keyword'}, 'value': { 'analyzer': 'standard', 'type': 'text', 'fields': { 'raw': {'type': 'keyword'}, 'folded': { 'analyzer': 'folding', 'type': 'text' } } } } } } } if create: se = SearchEngineFactory().create() se.create_index(index='concepts', body=index_settings) return index_settings
def prepare_term_index(create=False): """ Creates the settings and mappings in Elasticsearch to support term search """ index_settings = { 'settings': { 'analysis': { 'analyzer': { 'folding': { 'tokenizer': 'standard', 'filter': ['lowercase', 'asciifolding'] } } } }, 'mappings': { 'value': { 'properties': { 'ids': { 'type': 'string', 'index': 'not_analyzed' }, 'context': { 'type': 'string', 'index': 'not_analyzed' }, 'term': { 'type': 'string', 'analyzer': 'standard', 'fields': { 'folded': { 'type': 'string', 'analyzer': 'folding' } } } } } } } if create: se = SearchEngineFactory().create() se.create_index(index='term', body=index_settings, ignore=400) return index_settings
def prepare_term_index(self, create=False): """ Creates the settings and mappings in Elasticsearch to support term search """ index_settings = { 'settings':{ 'analysis': { 'analyzer': { 'folding': { 'tokenizer': 'standard', 'filter': [ 'lowercase', 'asciifolding' ] } } } }, 'mappings':{ 'value':{ 'properties': { 'ids':{'type': 'string', 'index' : 'not_analyzed'}, 'context':{'type': 'string', 'index' : 'not_analyzed'}, 'term': { 'type': 'string', 'analyzer': 'standard', 'fields': { 'folded': { 'type': 'string', 'analyzer': 'folding' } } } } } } } if create: se = SearchEngineFactory().create() se.create_index(index='term', body=index_settings, ignore=400) return index_settings
def prepare_resource_relations_index(self, create=False): """ Creates the settings and mappings in Elasticsearch to support related resources """ index_settings = { 'mappings': { 'all': { 'properties': { 'resourcexid': { 'type': 'long' }, 'notes': { 'type': 'string' }, 'relationshiptype': { 'type': 'string', 'index': 'not_analyzed' }, 'entityid2': { 'type': 'string', 'index': 'not_analyzed' }, 'entityid1': { 'type': 'string', 'index': 'not_analyzed' } } } } } if create: se = SearchEngineFactory().create() se.create_index(index='resource_relations', body=index_settings, ignore=400) return index_settings
def prepare_search_index(resource_model_id, create=False): """ Creates the settings and mappings in Elasticsearch to support resource search """ index_settings = { 'settings': { 'analysis': { 'analyzer': { 'folding': { 'tokenizer': 'standard', 'filter': ['lowercase', 'asciifolding'] } } } }, 'mappings': { resource_model_id: { 'properties': { 'graphid': { 'type': 'keyword' }, 'resourceinstanceid': { 'type': 'keyword' }, 'root_ontology_class': { 'type': 'keyword' }, 'displayname': { 'type': 'keyword' }, 'displaydescription': { 'type': 'keyword' }, 'map_popup': { 'type': 'keyword' }, 'provisional': { 'type': 'keyword' }, 'tiles': { 'type': 'nested', 'properties': { "tiles": { 'enabled': False }, 'tileid': { 'type': 'keyword' }, 'nodegroup_id': { 'type': 'keyword' }, 'parenttile_id': { 'type': 'keyword' }, 'resourceinstanceid_id': { 'type': 'keyword' } } }, 'strings': { 'type': 'nested', 'properties': { 'string': { 'type': 'text', 'index': 'analyzed', 'fields': { 'raw': { 'type': 'keyword' }, 'folded': { 'type': 'text', 'analyzer': 'folding' } } }, 'nodegroup_id': { 'type': 'keyword' }, 'provisional': { 'type': 'keyword' } } }, 'domains': { 'type': 'nested', 'properties': { 'value': { 'type': 'text', 'index': 'analyzed', 'fields': { 'raw': { 'type': 'keyword' } } }, 'conceptid': { 'type': 'keyword' }, 'valueid': { 'type': 'keyword' }, 'nodegroup_id': { 'type': 'keyword' }, 'provisional': { 'type': 'keyword' } } }, 'geometries': { 'type': 'nested', 'properties': { 'geom': { 'properties': { 'features': { 'properties': { 'geometry': { 'type': 'geo_shape' }, 'id': { 'type': 'keyword' }, 'type': { 'type': 'keyword' }, 'properties': { 'enabled': False } } }, 'type': { 'type': 'keyword' } } }, 'nodegroup_id': { 'type': 'keyword' }, 'provisional': { 'type': 'keyword' } } }, 'points': { 'type': 'nested', 'properties': { 'point': { 'type': 'geo_point' }, 'nodegroup_id': { 'type': 'keyword' }, 'provisional': { 'type': 'keyword' } } }, 'dates': { 'type': 'nested', 'properties': { 'date': { 'type': 'float' }, 'nodegroup_id': { 'type': 'keyword' }, 'nodeid': { 'type': 'keyword' }, 'provisional': { 'type': 'keyword' } } }, 'numbers': { 'type': 'nested', 'properties': { 'number': { 'type': 'double' }, 'nodegroup_id': { 'type': 'keyword' }, 'provisional': { 'type': 'keyword' } } }, 'date_ranges': { 'type': 'nested', 'properties': { 'date_range': { 'type': 'float_range' }, 'nodegroup_id': { 'type': 'keyword' }, 'provisional': { 'type': 'keyword' } } } } } } } if create: se = SearchEngineFactory().create() try: se.create_index(index='resource', body=index_settings) except: index_settings = index_settings['mappings'] se.create_mapping(index='resource', doc_type=resource_model_id, body=index_settings) return index_settings
def prepare_search_index(resource_model_id, create=False): """ Creates the settings and mappings in Elasticsearch to support resource search """ index_settings = { 'settings':{ 'analysis': { 'analyzer': { 'folding': { 'tokenizer': 'standard', 'filter': [ 'lowercase', 'asciifolding' ] } } } }, 'mappings': { resource_model_id : { 'properties' : { 'graphid': {'type' : 'string', 'index' : 'not_analyzed'}, 'resourceinstanceid': {'type' : 'string', 'index' : 'not_analyzed'}, 'primaryname': {'type' : 'string', 'index' : 'not_analyzed'}, 'tiles' : { 'type' : 'nested', 'properties' : { "tiles": {'enabled': False}, 'tileid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'nodegroup_id' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parenttile_id' : {'type' : 'string', 'index' : 'not_analyzed'}, 'resourceinstanceid_id' : {'type' : 'string', 'index' : 'not_analyzed'} } }, 'strings' : { 'type' : 'string', 'index' : 'analyzed', 'fields' : { 'raw' : { 'type' : 'string', 'index' : 'not_analyzed'}, 'folded': { 'type': 'string', 'analyzer': 'folding'} } }, 'domains' : { 'properties' : { 'value' : { 'type' : 'string', 'index' : 'analyzed', 'fields' : { 'raw' : { 'type' : 'string', 'index' : 'not_analyzed'} } }, 'conceptid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'valueid' : {'type' : 'string', 'index' : 'not_analyzed'}, } }, 'geometries' : { "properties": { "features": { "properties": { "geometry": {"type": "geo_shape"}, "id": { 'type' : 'string', 'index' : 'not_analyzed'}, "type": { 'type' : 'string', 'index' : 'not_analyzed'}, "properties": { "enabled": False } } }, "type": { 'type' : 'string', 'index' : 'not_analyzed'} } }, 'dates' : { "type" : "date" }, 'numbers' : { "type" : "double" } } } } } if create: se = SearchEngineFactory().create() try: se.create_index(index='resource', body=index_settings) except: index_settings = index_settings['mappings'] se.create_mapping(index='resource', doc_type=resource_model_id, body=index_settings) return index_settings
def tearDownClass(cls): se = SearchEngineFactory().create() se.delete_index(index='strings') se.create_index(index='strings')
def prepare_search_index(resource_model_id, create=False): """ Creates the settings and mappings in Elasticsearch to support resource search """ index_settings = { 'settings': { 'analysis': { 'analyzer': { 'folding': { 'tokenizer': 'standard', 'filter': [ 'lowercase', 'asciifolding' ] } } } }, 'mappings': { resource_model_id : { 'properties' : { 'graphid': {'type': 'keyword'}, 'resourceinstanceid': {'type': 'keyword'}, 'root_ontology_class': {'type':'keyword'}, 'displayname': {'type': 'keyword'}, 'displaydescription': {'type': 'keyword'}, 'map_popup': {'type': 'keyword'}, 'provisional': {'type': 'keyword'}, 'tiles' : { 'type' : 'nested', 'properties' : { "tiles": {'enabled': False}, 'tileid' : {'type': 'keyword'}, 'nodegroup_id' : {'type': 'keyword'}, 'parenttile_id' : {'type': 'keyword'}, 'resourceinstanceid_id' : {'type': 'keyword'} } }, 'strings' : { 'type' : 'nested', 'properties': { 'string': { 'type' : 'text', 'index' : 'analyzed', 'fields' : { 'raw' : {'type': 'keyword'}, 'folded': { 'type': 'text', 'analyzer': 'folding'} } }, 'nodegroup_id' : {'type': 'keyword'}, 'provisional': {'type': 'keyword'} } }, 'domains' : { 'type' : 'nested', 'properties' : { 'value' : { 'type' : 'text', 'index' : 'analyzed', 'fields' : { 'raw' : {'type': 'keyword'} } }, 'conceptid' : {'type': 'keyword'}, 'valueid' : {'type': 'keyword'}, 'nodegroup_id' : {'type': 'keyword'}, 'provisional': {'type': 'keyword'} } }, 'geometries' : { 'type' : 'nested', 'properties': { 'geom': { 'properties': { 'features': { 'properties': { 'geometry': {'type': 'geo_shape'}, 'id': {'type': 'keyword'}, 'type': {'type': 'keyword'}, 'properties': { 'enabled': False } } }, 'type': {'type': 'keyword'} } }, 'nodegroup_id' : {'type': 'keyword'}, 'provisional': {'type': 'keyword'} } }, 'points': { 'type' : 'nested', 'properties' : { 'point' : {'type': 'geo_point'}, 'nodegroup_id' : {'type': 'keyword'}, 'provisional': {'type': 'keyword'} } }, 'dates' : { 'type' : 'nested', 'properties' : { 'date' : {'type': 'float'}, 'nodegroup_id' : {'type': 'keyword'}, 'nodeid' : {'type': 'keyword'}, 'provisional': {'type': 'keyword'} } }, 'numbers' : { 'type' : 'nested', 'properties' : { 'number' : {'type': 'double'}, 'nodegroup_id' : {'type': 'keyword'}, 'provisional': {'type': 'keyword'} } }, 'date_ranges': { 'type' : 'nested', 'properties' : { 'date_range' : {'type': 'float_range'}, 'nodegroup_id' : {'type': 'keyword'}, 'provisional': {'type': 'keyword'} } } } } } } if create: se = SearchEngineFactory().create() try: se.create_index(index='resource', body=index_settings) except: index_settings = index_settings['mappings'] se.create_mapping(index='resource', doc_type=resource_model_id, body=index_settings) return index_settings
def base_prepare_search_index(self, resource_type_id, create=False): """ Creates the settings and mappings in Elasticsearch to support resource search """ index_settings = { 'settings':{ 'analysis': { "filter": { "desc_ngram": { "type": "ngram", "min_gram": 2, "max_gram": 30 } }, 'analyzer': { 'folding': { 'tokenizer': 'standard', 'filter': [ 'lowercase', 'asciifolding'] }, 'ducet_sort': { 'tokenizer': 'keyword', 'filter': [ 'icu_collation'] }, "index_ngram": { "type": "custom", "tokenizer": "keyword", "filter": [ "desc_ngram", "lowercase" ] }, "search_ngram": { "type": "custom", "tokenizer": "keyword", "filter": ["standard", "lowercase"] } } } }, 'mappings': { resource_type_id : { 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'primaryname': {'type' : 'string', 'index' : 'not_analyzed'}, 'child_entities' : { 'type' : 'nested', 'index' : 'analyzed', 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'analyzer': 'ducet_sort'}, 'value' : { 'type' : 'string', "index_analyzer": "index_ngram", "search_analyzer": "search_ngram", 'fields' : { 'raw' : { 'type' : 'string', 'index' : 'not_analyzed'}, 'folded': { 'type': 'string', 'analyzer': 'folding'} } } } }, 'domains' : { 'type' : 'nested', 'index' : 'analyzed', 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'analyzer': 'ducet_sort'}, 'value' : { 'type' : 'string', 'index' : 'analyzed', 'fields' : { 'raw' : { 'type' : 'string', 'index' : 'not_analyzed'} } }, 'conceptid' : {'type' : 'string', 'index' : 'not_analyzed'}, } }, 'geometries' : { 'type' : 'nested', 'index' : 'analyzed', 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : { "type": "geo_shape" } } }, 'dates' : { 'type' : 'nested', 'index' : 'analyzed', 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : { "type" : "date" } } } } } } } if create: se = SearchEngineFactory().create() try: se.create_index(index='entity', body=index_settings) except: index_settings = index_settings['mappings'] se.create_mapping(index='entity', doc_type=resource_type_id, body=index_settings) return index_settings
def prepare_term_index(create=False): """ Creates the settings and mappings in Elasticsearch to support term search """ index_settings = { 'settings': { 'analysis': { 'analyzer': { 'folding': { 'tokenizer': 'standard', 'filter': [ 'lowercase', 'asciifolding' ] } } } }, 'mappings': { 'term': { 'properties': { 'nodegroupid': {'type': 'keyword'}, 'tileid': {'type': 'keyword'}, 'nodeid': {'type': 'keyword'}, 'resourceinstanceid': {'type': 'keyword'}, 'provisional': {'type': 'keyword'}, 'value': { 'analyzer': 'standard', 'type': 'text', 'fields': { 'raw': {'type': 'keyword'}, 'folded': { 'analyzer': 'folding', 'type': 'text' } } } } }, 'concept': { 'properties': { 'top_concept': {'type': 'keyword'}, 'conceptid': {'type': 'keyword'}, 'language': {'type': 'keyword'}, 'id': {'type': 'keyword'}, 'category': {'type': 'keyword'}, 'provisional': {'type': 'keyword'}, 'type': {'type': 'keyword'}, 'value': { 'analyzer': 'standard', 'type': 'text', 'fields': { 'raw': {'type': 'keyword'}, 'folded': { 'analyzer': 'folding', 'type': 'text' } } } } } } } if create: se = SearchEngineFactory().create() se.create_index(index='strings', body=index_settings) return index_settings
class BaseIndex(object): def __init__(self, index_name=None): if index_name is None or index_name == "": raise SearchIndexError("Index name is not defined") self.se = SearchEngineFactory().create() self.index_metadata = None self.index_name = index_name def prepare_index(self): """ Defines the Elastic Search mapping and settings for an index Arguments: None Keyword Arguments: None Return: None """ if self.index_metadata is not None: self.se.create_index(index=self.index_name, body=self.index_metadata) else: raise SearchIndexError("No index metadata defined.") def get_documents_to_index(self, resourceinstance, tiles): """ Gets a document to index into Elastic Search Arguments: resourceinstance -- resource instance object tiles -- list of tiles that make up the resource instance Keyword Arguments: None Return: tuple of (document, document id) """ raise NotImplementedError def index_document(self, document=None, id=None): """ Indexes a document into Elastic Search Arguments: None Keyword Arguments: document -- the document to index id -- the id of the document Return: None """ if document is not None and id is not None: self.se.index_data(index=self.index_name, body=document, id=id) def index_resources(self, resources=None, batch_size=settings.BULK_IMPORT_BATCH_SIZE, quiet=False): """ Indexes a list of resources in bulk to Elastic Search Keyword Arguments: resources -- the list of resource instances to index batch_size -- the number of records to index as a group, the larger the number to more memory required quiet -- Silences the status bar output during certain operations, use in celery operations for example Return: None """ start = datetime.now() q = Query(se=self.se) self.se.refresh(index=self.index_name) count_before = self.se.count(index=self.index_name, body=q.dsl) result_summary = {"database": len(resources), "indexed": 0} if quiet is False: bar = pyprind.ProgBar(len(resources), bar_char="█") if len(resources) > 1 else None with self.se.BulkIndexer(batch_size=batch_size, refresh=True) as indexer: for resource in resources: if quiet is False and bar is not None: bar.update(item_id=resource) tiles = list(models.TileModel.objects.filter(resourceinstance=resource)) document, doc_id = self.get_documents_to_index(resource, tiles) if document is not None and id is not None: indexer.add(index=self.index_name, id=doc_id, data=document) self.se.refresh(index=self.index_name) result_summary["indexed"] = self.se.count(index=self.index_name, body=q.dsl) - count_before status = "Passed" if result_summary["database"] == result_summary["indexed"] else "Failed" print(f"Custom Index - {settings.ELASTICSEARCH_PREFIX}_{self.index_name}") print( f" Status: {status}, In Database: {result_summary['database']}, Indexed: {result_summary['indexed']}, Took: {(datetime.now() - start).seconds} seconds" ) def delete_resources(self, resources=None): """ Deletes documents from an index based on the passed in list of resources Delete by query, so this is a single operation Keyword Arguments: resources -- a single resource instance or a list of resource instances """ q = Query(se=self.se) if not isinstance(resources, list): resourcelist = [resources] else: resourcelist = resources list_of_ids_to_delete = [] for resource in resourcelist: list_of_ids_to_delete.append(resource.pk) ids_query = Ids(ids=list_of_ids_to_delete) q.add_query(ids_query) q.delete(index=self.index_name) def delete_index(self): """ Deletes this index from Elastic Search Arguments: None Keyword Arguments: None Return: None """ self.se.delete_index(index=self.index_name) def reindex(self, graphids=None, clear_index=True, batch_size=settings.BULK_IMPORT_BATCH_SIZE, quiet=False): """ Reindexes the index. By default this does nothing, it needs to be implemented in a subclass. By default you can pass in a list of graph ids to trigger the reindex. This will loop through all resource instances of each graph type. Example subclass command: def reindex(self, clear_index=True): PARCEL_GRAPHID = "e3c35dca-5e72-11ea-a2d3-dca90488358a" super(CustomIndexName, self).reindex(graphids=[PARCEL_GRAPHID], clear_index=clear_index) Keyword Arguments: graphids -- list of graphs ids to trigger the reindex on, will get all resource instances of each graph id supplied clear_index -- True(default) to clear all documents out of the index before reindexing begins batch_size -- the number of records to index as a group, the larger the number to more memory required Return: None """ if graphids is not None: if clear_index: self.delete_index() self.prepare_index() for graphid in graphids: resources = Resource.objects.filter(graph_id=graphid) self.index_resources(resources=resources, batch_size=batch_size, quiet=quiet) else: raise NotImplementedError
class BaseIndex(object): def __init__(self, index_name=None): if index_name is None or index_name is "": raise SearchIndexError("Index name is not defined") self.se = SearchEngineFactory().create() self.index_metadata = None self.index_name = index_name def prepare_index(self): """ Defines the Elastic Search mapping and settings for an index Arguments: None Keyword Arguments: None Return: None """ if self.index_metadata is not None: self.se.create_index(index=self.index_name, body=self.index_metadata) else: raise SearchIndexError("No index metadata defined.") def get_documents_to_index(self, resourceinstance, tiles): """ Gets a document to index into Elastic Search Arguments: resourceinstance -- resource instance object tiles -- list of tiles that make up the resource instance Keyword Arguments: None Return: tuple of (document, document id) """ raise NotImplementedError def index_document(self, document=None, id=None): """ Indexes a document into Elastic Search Arguments: None Keyword Arguments: document -- the document to index id -- the id of the document Return: None """ if document is not None and id is not None: self.se.index_data(index=self.index_name, body=document, id=id) def bulk_index(self, resources=None, resource_type=None, graph_name=None, clear_index=True): """ Indexes a list of documents in bulk to Elastic Search Arguments: None Keyword Arguments: resources -- the list of resource instances to index resource_type -- the type of resources being indexed graph_name -- the name of the graph model that represents the resources being indexed clear_index -- True(default) to remove all index records of type "resource_type" before indexing, assumes that a field called "graph_id" exists on the indexed documents Return: None """ start = datetime.now() q = Query(se=self.se) if clear_index: term = Term(field="graph_id", term=str(resource_type)) q.add_query(term) q.delete(index=self.index_name, refresh=True) q = Query(se=self.se) count_before = self.se.count(index=self.index_name, body=q.dsl) result_summary = {"database": len(resources), "indexed": 0} with self.se.BulkIndexer(batch_size=settings.BULK_IMPORT_BATCH_SIZE, refresh=True) as indexer: for resource in resources: tiles = list( models.TileModel.objects.filter(resourceinstance=resource)) document, doc_id = self.get_documents_to_index(resource, tiles) if document is not None and id is not None: indexer.add(index=self.index_name, id=doc_id, data=document) result_summary["indexed"] = self.se.count(index=self.index_name, body=q.dsl) - count_before status = "Passed" if result_summary["database"] == result_summary[ "indexed"] else "Failed" print("Custom Index - %s:" % self.index_name) print( " Status: {0}, Resource Type: {1}, In Database: {2}, Indexed: {3}, Took: {4} seconds" .format(status, graph_name, result_summary["database"], result_summary["indexed"], (datetime.now() - start).seconds)) def delete_index(self): """ Deletes this index from Elastic Search Arguments: None Keyword Arguments: None Return: None """ self.se.delete_index(index=self.index_name)
def setUpClass(cls): se = SearchEngineFactory().create() se.delete_index(index='terms,concepts') se.create_index(index='terms,concepts')
def prepare_search_index(self, resource_type_id, create=False): """ Creates the settings and mappings in Elasticsearch to support resource search """ index_settings = { 'settings':{ 'analysis': { 'analyzer': { 'folding': { 'tokenizer': 'standard', 'filter': [ 'lowercase', 'asciifolding' ] } } } }, 'mappings': { resource_type_id : { 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'primaryname': {'type' : 'string', 'index' : 'not_analyzed'}, 'child_entities' : { 'type' : 'nested', 'index' : 'analyzed', 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : { 'type' : 'string', 'index' : 'analyzed', 'fields' : { 'raw' : { 'type' : 'string', 'index' : 'not_analyzed'}, 'folded': { 'type': 'string', 'analyzer': 'folding'} } } } }, 'domains' : { 'type' : 'nested', 'index' : 'analyzed', 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : { 'type' : 'string', 'index' : 'analyzed', 'fields' : { 'raw' : { 'type' : 'string', 'index' : 'not_analyzed'} } }, 'conceptid' : {'type' : 'string', 'index' : 'not_analyzed'}, } }, 'geometries' : { 'type' : 'nested', 'index' : 'analyzed', 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : { "type": "geo_shape" } } }, 'dates' : { 'type' : 'nested', 'index' : 'analyzed', 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : { "type" : "date" } } }, 'extendeddates' : { 'type' : 'nested', 'index' : 'analyzed', 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : { 'type' : 'integer' } } }, 'extendeddategroups' : { 'properties' : { 'conceptid': {'type' : 'string', 'index' : 'not_analyzed'}, 'value': {'type' : 'integer', 'index' : 'not_analyzed'} } } } } } } if create: se = SearchEngineFactory().create() try: se.create_index(index='entity', body=index_settings) except: index_settings = index_settings['mappings'] se.create_mapping(index='entity', doc_type=resource_type_id, body=index_settings) return index_settings
def prepare_search_index(self, resource_type_id, create=False): """ Creates the settings and mappings in Elasticsearch to support resource search """ index_settings = { 'settings':{ 'analysis': { 'analyzer': { 'folding': { 'tokenizer': 'standard', 'filter': [ 'lowercase', 'asciifolding' ] } } } }, 'mappings': { resource_type_id : { 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'primaryname': {'type' : 'string', 'index' : 'not_analyzed'}, 'child_entities' : { 'type' : 'nested', 'index' : 'analyzed', 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : { 'type' : 'string', 'index' : 'analyzed', 'fields' : { 'raw' : { 'type' : 'string', 'index' : 'not_analyzed'}, 'folded': { 'type': 'string', 'analyzer': 'folding'} } } } }, 'domains' : { 'type' : 'nested', 'index' : 'analyzed', 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : { 'type' : 'string', 'index' : 'analyzed', 'fields' : { 'raw' : { 'type' : 'string', 'index' : 'not_analyzed'} } }, 'conceptid' : {'type' : 'string', 'index' : 'not_analyzed'}, } }, 'geometries' : { 'type' : 'nested', 'index' : 'analyzed', 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : { "type": "geo_shape" } } }, 'dates' : { 'type' : 'nested', 'index' : 'analyzed', 'properties' : { 'entityid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'parentid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'property' : {'type' : 'string', 'index' : 'not_analyzed'}, 'entitytypeid' : {'type' : 'string', 'index' : 'not_analyzed'}, 'businesstablename' : {'type' : 'string', 'index' : 'not_analyzed'}, 'label' : {'type' : 'string', 'index' : 'not_analyzed'}, 'value' : { "type" : "date" } } } } } } } if create: se = SearchEngineFactory().create() try: se.create_index(index='entity', body=index_settings) except: index_settings = index_settings['mappings'] se.create_mapping(index='entity', doc_type=resource_type_id, body=index_settings) return index_settings
def tearDownClass(cls): se = SearchEngineFactory().create() se.delete_index(index='concept_labels') se.delete_index(index='term') se.create_index(index='concept_labels') se.create_index(index='term')
def prepare_search_index(create=False): """ Creates the settings and mappings in Elasticsearch to support resource search """ index_settings = { "settings": { "analysis": { "analyzer": { "folding": { "tokenizer": "standard", "filter": ["lowercase", "asciifolding"] } } } }, "mappings": { "_doc": { "properties": { "graph_id": { "type": "keyword" }, "legacyid": { "type": "text", "fields": { "keyword": { "ignore_above": 256, "type": "keyword" } } }, "resourceinstanceid": { "type": "keyword" }, "root_ontology_class": { "type": "keyword" }, "displayname": { "type": "keyword" }, "displaydescription": { "type": "keyword" }, "map_popup": { "type": "keyword" }, "provisional_resource": { "type": "keyword" }, "tiles": { "type": "nested", "properties": { "tiles": { "enabled": False }, "tileid": { "type": "keyword" }, "nodegroup_id": { "type": "keyword" }, "parenttile_id": { "type": "keyword" }, "resourceinstanceid_id": { "type": "keyword" }, "provisionaledits": { "enabled": False }, }, }, "strings": { "type": "nested", "properties": { "string": { "type": "text", "fields": { "raw": { "type": "keyword" }, "folded": { "type": "text", "analyzer": "folding" } }, }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, "ids": { "type": "nested", "properties": { "id": { "type": "keyword" }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" } }, }, "domains": { "type": "nested", "properties": { "value": { "type": "text", "fields": { "raw": { "type": "keyword" } } }, "conceptid": { "type": "keyword" }, "valueid": { "type": "keyword" }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, "geometries": { "type": "nested", "properties": { "geom": { "properties": { "features": { "properties": { "geometry": { "type": "geo_shape" }, "id": { "type": "keyword" }, "type": { "type": "keyword" }, "properties": { "enabled": False }, } }, "type": { "type": "keyword" }, } }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, "points": { "type": "nested", "properties": { "point": { "type": "geo_point" }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, "dates": { "type": "nested", "properties": { "date": { "type": "float" }, "nodegroup_id": { "type": "keyword" }, "nodeid": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, "numbers": { "type": "nested", "properties": { "number": { "type": "double" }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, "date_ranges": { "type": "nested", "properties": { "date_range": { "type": "float_range" }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, } } }, } if create: se = SearchEngineFactory().create() se.create_index(index="resources", body=index_settings) return index_settings
def tearDownClass(cls): se = SearchEngineFactory().create() se.delete_index(index="concept_labels") se.delete_index(index="term") se.create_index(index="concept_labels") se.create_index(index="term")
def tearDownClass(cls): se = SearchEngineFactory().create() se.delete_index(index='strings') se.create_index(index='strings')
def tearDownClass(cls): se = SearchEngineFactory().create() se.delete_index(index="terms,concepts") se.create_index(index="terms,concepts")
def prepare_search_index(create=False): """ Creates the settings and mappings in Elasticsearch to support resource search """ index_settings = { "settings": { "analysis": { "analyzer": { "folding": { "tokenizer": "standard", "filter": ["lowercase", "asciifolding"] } } }, "index.mapping.total_fields.limit": 20000, }, "mappings": { "_doc": { "properties": { "graph_id": { "type": "keyword" }, "legacyid": { "type": "text", "fields": { "keyword": { "ignore_above": 256, "type": "keyword" } } }, "resourceinstanceid": { "type": "keyword" }, "root_ontology_class": { "type": "keyword" }, "displayname": { "type": "keyword" }, "displaydescription": { "type": "keyword" }, "map_popup": { "type": "keyword" }, "provisional_resource": { "type": "keyword" }, "tiles": { "type": "nested", "properties": { "tiles": { "enabled": False }, "tileid": { "type": "keyword" }, "nodegroup_id": { "type": "keyword" }, "parenttile_id": { "type": "keyword" }, "resourceinstanceid_id": { "type": "keyword" }, "provisionaledits": { "enabled": False }, "data": { "properties": {} }, }, }, "permissions": { "type": "nested", "properties": { "users_without_read_perm": { "type": "integer" }, "users_without_edit_perm": { "type": "integer" }, "users_without_delete_perm": { "type": "integer" }, "users_with_no_access": { "type": "integer" }, }, }, "strings": { "type": "nested", "properties": { "string": { "type": "text", "fields": { "raw": { "type": "keyword" }, "folded": { "type": "text", "analyzer": "folding" } }, }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, "ids": { "type": "nested", "properties": { "id": { "type": "keyword" }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" } }, }, "domains": { "type": "nested", "properties": { "value": { "type": "text", "fields": { "raw": { "type": "keyword" } } }, "conceptid": { "type": "keyword" }, "valueid": { "type": "keyword" }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, "geometries": { "type": "nested", "properties": { "geom": { "properties": { "features": { "properties": { "geometry": { "type": "geo_shape" }, "id": { "type": "keyword" }, "type": { "type": "keyword" }, "properties": { "enabled": False }, } }, "type": { "type": "keyword" }, } }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, "points": { "type": "nested", "properties": { "point": { "type": "geo_point" }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, "dates": { "type": "nested", "properties": { "date": { "type": "integer" }, "nodegroup_id": { "type": "keyword" }, "nodeid": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, "numbers": { "type": "nested", "properties": { "number": { "type": "double" }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, "date_ranges": { "type": "nested", "properties": { "date_range": { "type": "integer_range" }, "nodegroup_id": { "type": "keyword" }, "provisional": { "type": "boolean" }, }, }, } } }, } try: datatype_factory = DataTypeFactory() data = index_settings["mappings"]["_doc"]["properties"]["tiles"][ "properties"]["data"]["properties"] for node in models.Node.objects.all(): datatype = datatype_factory.get_instance(node.datatype) datatype_mapping = datatype.default_es_mapping() if datatype_mapping and datatype_factory.datatypes[ node.datatype].defaultwidget: data[str(node.nodeid)] = datatype_mapping except ProgrammingError: print( "Skipping datatype mappings because the datatypes table is not yet available" ) if create: se = SearchEngineFactory().create() se.create_index(index="resources", body=index_settings) return index_settings