def post_resource_registry(self, config):
for res in RT.keys():
rt = IonObject("ResourceType", name=res)
def post_resource_registry(self, config):
for res in RT.keys():
rt = IonObject("ResourceType", name=res)
def index_bootstrap(self):
'''
Creates the initial set of desired indexes based on a standard definition
'''
#=======================================================================================
# Create the _river index based on the cluster configurations
#=======================================================================================
IndexManagementService._es_call(
self.es.index_create,
'_river',
number_of_shards = self.river_shards,
number_of_replicas = self.river_replicas
)
filters = {
'_id' : '_design/filters',
'filters' : {
}
}
#=======================================================================================
# For each of the resource types in the list of values for each standard index,
# create a mapping and a context type in ElasticSearch based on the searchable fields.
#=======================================================================================
for k,v in STD_INDEXES.iteritems():
response = IndexManagementService._es_call(
self.es.index_create,
k,
number_of_shards=self.index_shards,
number_of_replicas=self.index_replicas
)
IndexManagementService._check_response(response)
body = 'function(doc, req) { switch(doc.type_) { default: return false; }}'
for res in v:
body = re.sub(r'default:', 'case "%s": return true; default:' % res, body)
mappings = self.es_mapping(res)
response = IndexManagementService._es_call(self.es.raw,'%s/%s/_mapping' %(k,res), 'POST', mappings)
IndexManagementService._check_response(response)
filters['filters'][k] = body
#=======================================================================================
# Get an instance of the datastore instance used to create the CouchDB filters
# in support of the ElasticSearch river's filter
# - Allows us to filter based on resource type
#=======================================================================================
cc = self.container
db = cc.datastore_manager.get_datastore('resources')
datastore_name = db.datastore_name
db = db.server[datastore_name]
db.create(filters)
#--------------------------------------------------------------------------------
# Create the river connection between CouchDB and ElasticSearch
#--------------------------------------------------------------------------------
for k,v in STD_INDEXES.iteritems():
response = IndexManagementService._es_call(self.es.river_couchdb_create,
index_name = k,
couchdb_db = datastore_name,
couchdb_host = CFG.server.couchdb.host,
couchdb_port = CFG.server.couchdb.port,
couchdb_user = CFG.server.couchdb.username,
couchdb_password = CFG.server.couchdb.password,
couchdb_filter = 'filters/%s' % k,
script= ELASTICSEARCH_CONTEXT_SCRIPT
)
IndexManagementService._check_response(response)
#=======================================================================================
# Create and map the edge indexes
#=======================================================================================
#--------------------------------------------------------------------------------
# Resources Index
#--------------------------------------------------------------------------------
response = IndexManagementService._es_call(self.es.index_create,'%s_resources_index' % self.sysname,
number_of_shards=self.index_shards,
number_of_replicas=self.index_replicas
)
IndexManagementService._check_response(response)
for t in RT.values():
mappings = self.es_mapping(t)
response = IndexManagementService._es_call(self.es.raw,'%s_resources_index/%s/_mapping' % (self.sysname,t), 'POST', mappings)
IndexManagementService._check_response(response)
response = IndexManagementService._es_call(self.es.river_couchdb_create,
index_name = '%s_resources_index' % self.sysname,
couchdb_db = datastore_name,
couchdb_host = CFG.server.couchdb.host,
couchdb_port = CFG.server.couchdb.port,
couchdb_user = CFG.server.couchdb.username,
couchdb_password = CFG.server.couchdb.password,
script= ELASTICSEARCH_CONTEXT_SCRIPT
)
IndexManagementService._check_response(response)
#--------------------------------------------------------------------------------
# Events Index
#--------------------------------------------------------------------------------
response = IndexManagementService._es_call(self.es.index_create,'%s_events_index' % self.sysname,
number_of_shards=self.index_shards,
number_of_replicas=self.index_replicas
)
IndexManagementService._check_response(response)
for event in get_events():
mappings = self.es_mapping(event)
response = IndexManagementService._es_call(self.es.raw, '%s_events_index/%s/_mapping' % (self.sysname, event), 'POST', mappings)
IndexManagementService._check_response(response)
response = IndexManagementService._es_call(self.es.river_couchdb_create,
index_name = '%s_events_index' % self.sysname,
couchdb_db = '%s_events' % self.sysname,
couchdb_host = CFG.server.couchdb.host,
couchdb_port = CFG.server.couchdb.port,
couchdb_user = CFG.server.couchdb.username,
couchdb_password = CFG.server.couchdb.password,
script= ELASTICSEARCH_CONTEXT_SCRIPT
)
IndexManagementService._check_response(response)
#=======================================================================================
# Construct the resources
#=======================================================================================
ims_cli = IndexManagementServiceClient()
#--------------------------------------------------------------------------------
# Standard Indexes
#--------------------------------------------------------------------------------
for index,resources in STD_INDEXES.iteritems():
ims_cli.create_index(
name=index,
description='%s ElasticSearch Index Resource' % index,
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=self.attr_mapping(resources)
)
#--------------------------------------------------------------------------------
# CouchDB Indexes
#--------------------------------------------------------------------------------
for index,datastore in COUCHDB_INDEXES.iteritems():
ims_cli.create_index(
name=index,
description='%s CouchDB Index Resource' % index,
content_type=IndexManagementService.COUCHDB_INDEX,
datastore_name=datastore
)
#--------------------------------------------------------------------------------
# Edge Indexes
#--------------------------------------------------------------------------------
ims_cli.create_index(
name='%s_resources_index' % self.sysname,
description='Resources Index',
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=self.attr_mapping(RT.keys())
)
ims_cli.create_index(
name='%s_events_index' % self.sysname,
description='Events Index',
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=self.attr_mapping(get_events())
)
def index_bootstrap(self):
'''
Creates the initial set of desired indexes based on a standard definition
'''
#=======================================================================================
# Create the _river index based on the cluster configurations
#=======================================================================================
IndexManagementService._es_call(self.es.index_create,
'_river',
number_of_shards=self.river_shards,
number_of_replicas=self.river_replicas)
filters = {'_id': '_design/filters', 'filters': {}}
#=======================================================================================
# For each of the resource types in the list of values for each standard index,
# create a mapping and a context type in ElasticSearch based on the searchable fields.
#=======================================================================================
for k, v in STD_INDEXES.iteritems():
response = IndexManagementService._es_call(
self.es.index_create,
k,
number_of_shards=self.index_shards,
number_of_replicas=self.index_replicas)
IndexManagementService._check_response(response)
body = 'function(doc, req) { switch(doc.type_) { default: return false; }}'
for res in v:
body = re.sub(r'default:',
'case "%s": return true; default:' % res, body)
mappings = self.es_mapping(res)
response = IndexManagementService._es_call(
self.es.raw, '%s/%s/_mapping' % (k, res), 'POST', mappings)
IndexManagementService._check_response(response)
filters['filters'][k] = body
#=======================================================================================
# Get an instance of the datastore instance used to create the CouchDB filters
# in support of the ElasticSearch river's filter
# - Allows us to filter based on resource type
#=======================================================================================
cc = self.container
db = cc.datastore_manager.get_datastore('resources')
datastore_name = db.datastore_name
db = db.server[datastore_name]
db.create(filters)
#--------------------------------------------------------------------------------
# Create the river connection between CouchDB and ElasticSearch
#--------------------------------------------------------------------------------
for k, v in STD_INDEXES.iteritems():
response = IndexManagementService._es_call(
self.es.river_couchdb_create,
index_name=k,
couchdb_db=datastore_name,
couchdb_host=CFG.server.couchdb.host,
couchdb_port=CFG.server.couchdb.port,
couchdb_user=CFG.server.couchdb.username,
couchdb_password=CFG.server.couchdb.password,
couchdb_filter='filters/%s' % k,
script=ELASTICSEARCH_CONTEXT_SCRIPT)
IndexManagementService._check_response(response)
#=======================================================================================
# Create and map the edge indexes
#=======================================================================================
#--------------------------------------------------------------------------------
# Resources Index
#--------------------------------------------------------------------------------
response = IndexManagementService._es_call(
self.es.index_create,
'%s_resources_index' % self.sysname,
number_of_shards=self.index_shards,
number_of_replicas=self.index_replicas)
IndexManagementService._check_response(response)
for t in RT.values():
mappings = self.es_mapping(t)
response = IndexManagementService._es_call(
self.es.raw,
'%s_resources_index/%s/_mapping' % (self.sysname, t), 'POST',
mappings)
IndexManagementService._check_response(response)
response = IndexManagementService._es_call(
self.es.river_couchdb_create,
index_name='%s_resources_index' % self.sysname,
couchdb_db=datastore_name,
couchdb_host=CFG.server.couchdb.host,
couchdb_port=CFG.server.couchdb.port,
couchdb_user=CFG.server.couchdb.username,
couchdb_password=CFG.server.couchdb.password,
script=ELASTICSEARCH_CONTEXT_SCRIPT)
IndexManagementService._check_response(response)
#--------------------------------------------------------------------------------
# Events Index
#--------------------------------------------------------------------------------
response = IndexManagementService._es_call(
self.es.index_create,
'%s_events_index' % self.sysname,
number_of_shards=self.index_shards,
number_of_replicas=self.index_replicas)
IndexManagementService._check_response(response)
for event in get_events():
mappings = self.es_mapping(event)
response = IndexManagementService._es_call(
self.es.raw,
'%s_events_index/%s/_mapping' % (self.sysname, event), 'POST',
mappings)
IndexManagementService._check_response(response)
response = IndexManagementService._es_call(
self.es.river_couchdb_create,
index_name='%s_events_index' % self.sysname,
couchdb_db='%s_events' % self.sysname,
couchdb_host=CFG.server.couchdb.host,
couchdb_port=CFG.server.couchdb.port,
couchdb_user=CFG.server.couchdb.username,
couchdb_password=CFG.server.couchdb.password,
script=ELASTICSEARCH_CONTEXT_SCRIPT)
IndexManagementService._check_response(response)
#=======================================================================================
# Construct the resources
#=======================================================================================
ims_cli = IndexManagementServiceClient()
#--------------------------------------------------------------------------------
# Standard Indexes
#--------------------------------------------------------------------------------
for index, resources in STD_INDEXES.iteritems():
ims_cli.create_index(
name=index,
description='%s ElasticSearch Index Resource' % index,
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=self.attr_mapping(resources))
#--------------------------------------------------------------------------------
# CouchDB Indexes
#--------------------------------------------------------------------------------
for index, datastore in COUCHDB_INDEXES.iteritems():
ims_cli.create_index(
name=index,
description='%s CouchDB Index Resource' % index,
content_type=IndexManagementService.COUCHDB_INDEX,
datastore_name=datastore)
#--------------------------------------------------------------------------------
# Edge Indexes
#--------------------------------------------------------------------------------
ims_cli.create_index(
name='%s_resources_index' % self.sysname,
description='Resources Index',
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=self.attr_mapping(RT.keys()))
ims_cli.create_index(
name='%s_events_index' % self.sysname,
description='Events Index',
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=self.attr_mapping(get_events()))
