def delete_indexes(self): ims_cli = IndexManagementServiceProcessClient(process=self) rr_cli = ResourceRegistryServiceProcessClient(process=self) #-------------------------------------------------------------------------------- # Standard Indexes #-------------------------------------------------------------------------------- for index,resources in STD_INDEXES.iteritems(): index_ids, _ = rr_cli.find_resources(name=index,restype=RT.ElasticSearchIndex, id_only=True) for index_id in index_ids: ims_cli.delete_index(index_id) #-------------------------------------------------------------------------------- # CouchDB Indexes #-------------------------------------------------------------------------------- for index,datastore in COUCHDB_INDEXES.iteritems(): index_ids, _ = rr_cli.find_resources(name=index, restype=RT.CouchDBIndex, id_only=True) for index_id in index_ids: ims_cli.delete_index(index_id) #-------------------------------------------------------------------------------- # Edge Indexes #-------------------------------------------------------------------------------- index_ids, _ = rr_cli.find_resources(name='%s_resources_index' % self.sysname,restype=RT.ElasticSearchIndex, id_only=True) for index_id in index_ids: ims_cli.delete_index(index_id) index_ids, _ = rr_cli.find_resources(name='%s_events_index' % self.sysname, restype=RT.ElasticSearchIndex, id_only=True) for index_id in index_ids: ims_cli.delete_index(index_id)
def delete_indexes(self): ims_cli = IndexManagementServiceProcessClient(process=self) rr_cli = ResourceRegistryServiceProcessClient(process=self) #-------------------------------------------------------------------------------- # Standard Indexes #-------------------------------------------------------------------------------- for index, resources in STD_INDEXES.iteritems(): index_ids, _ = rr_cli.find_resources(name=index, restype=RT.ElasticSearchIndex, id_only=True) for index_id in index_ids: ims_cli.delete_index(index_id) #-------------------------------------------------------------------------------- # CouchDB Indexes #-------------------------------------------------------------------------------- for index, datastore in COUCHDB_INDEXES.iteritems(): index_ids, _ = rr_cli.find_resources(name=index, restype=RT.CouchDBIndex, id_only=True) for index_id in index_ids: ims_cli.delete_index(index_id) #-------------------------------------------------------------------------------- # Edge Indexes #-------------------------------------------------------------------------------- index_ids, _ = rr_cli.find_resources(name='%s_resources_index' % self.sysname, restype=RT.ElasticSearchIndex, id_only=True) for index_id in index_ids: ims_cli.delete_index(index_id) index_ids, _ = rr_cli.find_resources(name='%s_events_index' % self.sysname, restype=RT.ElasticSearchIndex, id_only=True) for index_id in index_ids: ims_cli.delete_index(index_id)
def index_bootstrap(self): ''' Creates the initial set of desired indexes based on a standard definition ''' #======================================================================================= # Create the _river index based on the cluster configurations #======================================================================================= IndexManagementService._es_call( self.es.index_create, '_river', number_of_shards = self.river_shards, number_of_replicas = self.river_replicas ) filters = { '_id' : '_design/filters', 'filters' : { } } #======================================================================================= # For each of the resource types in the list of values for each standard index, # create a mapping and a context type in ElasticSearch based on the searchable fields. #======================================================================================= for k,v in STD_INDEXES.iteritems(): response = IndexManagementService._es_call( self.es.index_create, k, number_of_shards=self.index_shards, number_of_replicas=self.index_replicas ) IndexManagementService._check_response(response) body = 'function(doc, req) { switch(doc.type_) { default: return false; }}' for res in v: body = re.sub(r'default:', 'case "%s": return true; default:' % res, body) mappings = self.es_mapping(res) response = IndexManagementService._es_call(self.es.raw,'%s/%s/_mapping' %(k,res), 'POST', mappings) IndexManagementService._check_response(response) filters['filters'][k] = body #======================================================================================= # Get an instance of the datastore instance used to create the CouchDB filters # in support of the ElasticSearch river's filter # - Allows us to filter based on resource type #======================================================================================= cc = self.container db = cc.datastore_manager.get_datastore('resources') datastore_name = db.datastore_name db = db.server[datastore_name] if '_design/filters' not in db: db.create(filters) #-------------------------------------------------------------------------------- # Create the river connection between CouchDB and ElasticSearch #-------------------------------------------------------------------------------- for k,v in STD_INDEXES.iteritems(): response = IndexManagementService._es_call(self.es.river_couchdb_create, index_name = k, couchdb_db = datastore_name, couchdb_host = CFG.server.couchdb.host, couchdb_port = CFG.server.couchdb.port, couchdb_user = CFG.server.couchdb.username, couchdb_password = CFG.server.couchdb.password, couchdb_filter = 'filters/%s' % k, script= ELASTICSEARCH_CONTEXT_SCRIPT ) IndexManagementService._check_response(response) #======================================================================================= # Create and map the edge indexes #======================================================================================= #-------------------------------------------------------------------------------- # Resources Index #-------------------------------------------------------------------------------- response = IndexManagementService._es_call(self.es.index_create,'%s_resources_index' % self.sysname, number_of_shards=self.index_shards, number_of_replicas=self.index_replicas ) IndexManagementService._check_response(response) for t in RT.values(): mappings = self.es_mapping(t) response = IndexManagementService._es_call(self.es.raw,'%s_resources_index/%s/_mapping' % (self.sysname,t), 'POST', mappings) IndexManagementService._check_response(response) response = IndexManagementService._es_call(self.es.river_couchdb_create, index_name = '%s_resources_index' % self.sysname, couchdb_db = datastore_name, couchdb_host = CFG.server.couchdb.host, couchdb_port = CFG.server.couchdb.port, couchdb_user = CFG.server.couchdb.username, couchdb_password = CFG.server.couchdb.password, script= ELASTICSEARCH_CONTEXT_SCRIPT ) IndexManagementService._check_response(response) #-------------------------------------------------------------------------------- # Events Index #-------------------------------------------------------------------------------- response = IndexManagementService._es_call(self.es.index_create,'%s_events_index' % self.sysname, number_of_shards=self.index_shards, number_of_replicas=self.index_replicas ) IndexManagementService._check_response(response) for event in get_events(): mappings = self.es_mapping(event) response = IndexManagementService._es_call(self.es.raw, '%s_events_index/%s/_mapping' % (self.sysname, event), 'POST', mappings) IndexManagementService._check_response(response) response = IndexManagementService._es_call(self.es.river_couchdb_create, index_name = '%s_events_index' % self.sysname, couchdb_db = '%s_events' % self.sysname, couchdb_host = CFG.server.couchdb.host, couchdb_port = CFG.server.couchdb.port, couchdb_user = CFG.server.couchdb.username, couchdb_password = CFG.server.couchdb.password, script= ELASTICSEARCH_CONTEXT_SCRIPT ) IndexManagementService._check_response(response) #======================================================================================= # Construct the resources #======================================================================================= ims_cli = IndexManagementServiceProcessClient(process=self) #-------------------------------------------------------------------------------- # Standard Indexes #-------------------------------------------------------------------------------- for index,resources in STD_INDEXES.iteritems(): ims_cli.create_index( name=index, description='%s ElasticSearch Index Resource' % index, content_type=IndexManagementService.ELASTICSEARCH_INDEX, options=self.attr_mapping(resources) ) #-------------------------------------------------------------------------------- # CouchDB Indexes #-------------------------------------------------------------------------------- for index,datastore in COUCHDB_INDEXES.iteritems(): ims_cli.create_index( name=index, description='%s CouchDB Index Resource' % index, content_type=IndexManagementService.COUCHDB_INDEX, datastore_name=datastore ) #-------------------------------------------------------------------------------- # Edge Indexes #-------------------------------------------------------------------------------- ims_cli.create_index( name='%s_resources_index' % self.sysname, description='Resources Index', content_type=IndexManagementService.ELASTICSEARCH_INDEX, options=self.attr_mapping(RT.keys()) ) ims_cli.create_index( name='%s_events_index' % self.sysname, description='Events Index', content_type=IndexManagementService.ELASTICSEARCH_INDEX, options=self.attr_mapping(get_events()) )
def index_bootstrap(self): ''' Creates the initial set of desired indexes based on a standard definition ''' #======================================================================================= # Create the _river index based on the cluster configurations #======================================================================================= IndexManagementService._es_call(self.es.index_create, '_river', number_of_shards=self.river_shards, number_of_replicas=self.river_replicas) filters = {'_id': '_design/filters', 'filters': {}} #======================================================================================= # For each of the resource types in the list of values for each standard index, # create a mapping and a context type in ElasticSearch based on the searchable fields. #======================================================================================= for k, v in STD_INDEXES.iteritems(): response = IndexManagementService._es_call( self.es.index_create, k, number_of_shards=self.index_shards, number_of_replicas=self.index_replicas) IndexManagementService._check_response(response) body = 'function(doc, req) { switch(doc.type_) { default: return false; }}' for res in v: body = re.sub(r'default:', 'case "%s": return true; default:' % res, body) mappings = self.es_mapping(res) response = IndexManagementService._es_call( self.es.raw, '%s/%s/_mapping' % (k, res), 'POST', mappings) IndexManagementService._check_response(response) filters['filters'][k] = body #======================================================================================= # Get an instance of the datastore instance used to create the CouchDB filters # in support of the ElasticSearch river's filter # - Allows us to filter based on resource type #======================================================================================= cc = self.container db = cc.datastore_manager.get_datastore('resources') datastore_name = db.datastore_name db = db.server[datastore_name] if '_design/filters' not in db: db.create(filters) #-------------------------------------------------------------------------------- # Create the river connection between CouchDB and ElasticSearch #-------------------------------------------------------------------------------- for k, v in STD_INDEXES.iteritems(): response = IndexManagementService._es_call( self.es.river_couchdb_create, index_name=k, couchdb_db=datastore_name, couchdb_host=CFG.server.couchdb.host, couchdb_port=CFG.server.couchdb.port, couchdb_user=CFG.server.couchdb.username, couchdb_password=CFG.server.couchdb.password, couchdb_filter='filters/%s' % k, script=ELASTICSEARCH_CONTEXT_SCRIPT) IndexManagementService._check_response(response) #======================================================================================= # Create and map the edge indexes #======================================================================================= #-------------------------------------------------------------------------------- # Resources Index #-------------------------------------------------------------------------------- response = IndexManagementService._es_call( self.es.index_create, '%s_resources_index' % self.sysname, number_of_shards=self.index_shards, number_of_replicas=self.index_replicas) IndexManagementService._check_response(response) for t in RT.values(): mappings = self.es_mapping(t) response = IndexManagementService._es_call( self.es.raw, '%s_resources_index/%s/_mapping' % (self.sysname, t), 'POST', mappings) IndexManagementService._check_response(response) response = IndexManagementService._es_call( self.es.river_couchdb_create, index_name='%s_resources_index' % self.sysname, couchdb_db=datastore_name, couchdb_host=CFG.server.couchdb.host, couchdb_port=CFG.server.couchdb.port, couchdb_user=CFG.server.couchdb.username, couchdb_password=CFG.server.couchdb.password, script=ELASTICSEARCH_CONTEXT_SCRIPT) IndexManagementService._check_response(response) #-------------------------------------------------------------------------------- # Events Index #-------------------------------------------------------------------------------- response = IndexManagementService._es_call( self.es.index_create, '%s_events_index' % self.sysname, number_of_shards=self.index_shards, number_of_replicas=self.index_replicas) IndexManagementService._check_response(response) for event in get_events(): mappings = self.es_mapping(event) response = IndexManagementService._es_call( self.es.raw, '%s_events_index/%s/_mapping' % (self.sysname, event), 'POST', mappings) IndexManagementService._check_response(response) response = IndexManagementService._es_call( self.es.river_couchdb_create, index_name='%s_events_index' % self.sysname, couchdb_db='%s_events' % self.sysname, couchdb_host=CFG.server.couchdb.host, couchdb_port=CFG.server.couchdb.port, couchdb_user=CFG.server.couchdb.username, couchdb_password=CFG.server.couchdb.password, script=ELASTICSEARCH_CONTEXT_SCRIPT) IndexManagementService._check_response(response) #======================================================================================= # Construct the resources #======================================================================================= ims_cli = IndexManagementServiceProcessClient(process=self) #-------------------------------------------------------------------------------- # Standard Indexes #-------------------------------------------------------------------------------- for index, resources in STD_INDEXES.iteritems(): ims_cli.create_index( name=index, description='%s ElasticSearch Index Resource' % index, content_type=IndexManagementService.ELASTICSEARCH_INDEX, options=self.attr_mapping(resources)) #-------------------------------------------------------------------------------- # CouchDB Indexes #-------------------------------------------------------------------------------- for index, datastore in COUCHDB_INDEXES.iteritems(): ims_cli.create_index( name=index, description='%s CouchDB Index Resource' % index, content_type=IndexManagementService.COUCHDB_INDEX, datastore_name=datastore) #-------------------------------------------------------------------------------- # Edge Indexes #-------------------------------------------------------------------------------- ims_cli.create_index( name='%s_resources_index' % self.sysname, description='Resources Index', content_type=IndexManagementService.ELASTICSEARCH_INDEX, options=self.attr_mapping(RT.keys())) ims_cli.create_index( name='%s_events_index' % self.sysname, description='Events Index', content_type=IndexManagementService.ELASTICSEARCH_INDEX, options=self.attr_mapping(get_events()))