def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.ims_cli = IndexManagementServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
self.index_name = 'test_index'
def setUp(self):
super(DiscoveryIntTest, self).setUp()
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
if use_es:
self.es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
self.es_port = CFG.get_safe('server.elasticsearch.port', '9200')
CFG.server.elasticsearch.shards = 1
CFG.server.elasticsearch.replicas = 0
CFG.server.elasticsearch.river_shards = 1
CFG.server.elasticsearch.river_replicas = 0
self.es = ep.ElasticSearch(
host=self.es_host,
port=self.es_port,
timeout=10,
verbose=True
)
op = DotDict(CFG)
op.op = 'clean_bootstrap'
self.container.spawn_process('index_bootstrap','ion.processes.bootstrap.index_bootstrap','IndexBootStrap', op)
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2dm.yml")
self.ims_cli = IndexManagementServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
self.index_name = "test_index"
def setUp(self):
super(DiscoveryIntTest, self).setUp()
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
if use_es:
self.es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
self.es_port = CFG.get_safe('server.elasticsearch.port', '9200')
CFG.server.elasticsearch.shards = 1
CFG.server.elasticsearch.replicas = 0
CFG.server.elasticsearch.river_shards = 1
CFG.server.elasticsearch.river_replicas = 0
self.es = ep.ElasticSearch(
host=self.es_host,
port=self.es_port,
timeout=10,
verbose=True
)
op = DotDict(CFG)
op.op = 'clean_bootstrap'
self.container.spawn_process('index_bootstrap','ion.processes.bootstrap.index_bootstrap','IndexBootStrap', op)
def setUp(self):
super(DiscoveryIntTest, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2dm.yml', config)
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
def setUp(self):
raise SkipTest("Not yet ported to Postgres")
super(DiscoveryIntTest, self).setUp()
config = DotDict()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
class IndexManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2dm.yml")
self.ims_cli = IndexManagementServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
self.index_name = "test_index"
def test_create_elasticsearch_index(self):
index_name = self.index_name
ims_cli = self.ims_cli
rr_cli = self.rr_cli
options = SearchOptions()
options.attribute_match = ["test_field"]
index_id = ims_cli.create_index(
name=index_name, content_type=IndexManagementService.ELASTICSEARCH_INDEX, options=options
)
index_result = self.rr_cli.read(index_id)
self.assertIsInstance(index_result, ElasticSearchIndex)
self.assertTrue(index_result.name == index_name)
# ======================================
# Clean up
# ======================================
rr_cli.delete(index_id)
def test_create_couchdb_index(self):
index_name = self.index_name
ims_cli = self.ims_cli
rr_cli = self.rr_cli
options = SearchOptions()
options.attribute_match = ["name"]
index_id = ims_cli.create_index(
index_name,
content_type=IndexManagementService.COUCHDB_INDEX,
options=options,
datastore_name="fake",
view_name="fake/by_fake",
)
index_result = self.rr_cli.read(index_id)
self.assertIsInstance(index_result, CouchDBIndex)
self.assertTrue(index_result.name == index_name)
# ======================================
# Clean up
# ======================================
rr_cli.delete(index_id)
def test_read_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
index = ims_cli.read_index(index_id)
self.assertIsInstance(index, Index)
self.assertTrue(index.name == index_name)
rr_cli.delete(index_id)
def test_delete_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
ims_cli.delete_index(index_id)
with self.assertRaises(NotFound):
rr_cli.delete(index_id)
def test_update_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
index = ims_cli.read_index(index_id)
index.name = "another"
ims_cli.update_index(index)
index = rr_cli.read(index_id)
self.assertTrue(index.name == "another")
def test_find_indexes(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
# ======================================
# Index Pool
# ======================================
indexes = [Index(name="first"), Index(name="second"), Index(name="third")]
id_pool = list()
for index in indexes:
id_pool.append(rr_cli.create(index)[0])
index_id = ims_cli.find_indexes(index_name="second")
index = ims_cli.read_index(index_id)
self.assertTrue(index.name == "second")
# ======================================
# Clean up
# ======================================
for index_id in id_pool:
rr_cli.delete(index_id)
def test_list_indexes(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
# ======================================
# Index Pool
# ======================================
indexes = [Index(name="first"), Index(name="second"), Index(name="third")]
id_pool = list()
for index in indexes:
id_pool.append(rr_cli.create(index)[0])
names = set(ims_cli.list_indexes().keys())
self.assertTrue(names == set(["first", "second", "third"]))
# ======================================
# Clean up
# ======================================
for index_id in id_pool:
rr_cli.delete(index_id)
def test_create_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
with self.assertRaises(BadRequest):
ims_cli.create_collection("failing_collection")
resources = [Resource(), Resource(), Resource()]
resources = [rr_cli.create(i)[0] for i in resources]
collection_id = ims_cli.create_collection("working_collection", resources)
collection = rr_cli.read(collection_id)
collection_resources = ims_cli.list_collection_resources(collection_id, id_only=True)
self.assertTrue(
set(collection_resources) == set(resources), "%s != %s" % (set(collection_resources), set(resources))
)
def test_read_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
collection = Collection(name="working_collection")
collection_id, _ = rr_cli.create(collection)
collection = ims_cli.read_collection(collection_id)
self.assertTrue(collection.name == "working_collection")
def test_update_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
collection = Collection(name="useful_collection")
collection_id, _ = rr_cli.create(collection)
collection = rr_cli.read(collection_id)
collection.name = "nub"
ims_cli.update_collection(collection)
collection = rr_cli.read(collection_id)
self.assertTrue(collection.name == "nub")
def test_delete_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
res = Resource()
res_id, rev = rr_cli.create(res)
collection_id = ims_cli.create_collection(name="test_collection", resources=[res_id])
ims_cli.delete_collection(collection_id)
with self.assertRaises(NotFound):
rr_cli.read(collection_id)
def test_list_collection_resources(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
# ========================================
# Resource Pool
# ========================================
resources = [
InformationResource(name="bean_counter"),
InformationResource(name="lunar_rock"),
InformationResource("aperature"),
InformationResource("lemons"),
]
resources = [rr_cli.create(i)[0] for i in resources]
collection = Collection(name="park_bench")
collection_id = ims_cli.create_collection(name="park_bench", resources=resources)
retval = ims_cli.list_collection_resources(collection_id, id_only=True)
retval.sort()
resources.sort()
self.assertTrue(retval == resources, "%s != %s" % (retval, resources))
def test_find_collection(self):
res_id, _ = self.rr_cli.create(Resource(name="test_res"))
collection_id = self.ims_cli.create_collection("test", [res_id])
retval = self.ims_cli.find_collection(collection_name="test")
self.assertTrue(retval[0] == collection_id)
retval = self.ims_cli.find_collection(resource_ids=[res_id])
self.assertTrue(retval[0] == collection_id)
class IndexManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.ims_cli = IndexManagementServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
self.index_name = 'test_index'
def test_create_elasticsearch_index(self):
index_name = self.index_name
ims_cli = self.ims_cli
rr_cli = self.rr_cli
options = SearchOptions()
options.attribute_match = ['test_field']
index_id = ims_cli.create_index(
name=index_name,
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=options
)
index_result = self.rr_cli.read(index_id)
self.assertIsInstance(index_result,ElasticSearchIndex)
self.assertTrue(index_result.name == index_name)
#======================================
# Clean up
#======================================
rr_cli.delete(index_id)
def test_create_couchdb_index(self):
index_name = self.index_name
ims_cli = self.ims_cli
rr_cli = self.rr_cli
options = SearchOptions()
options.attribute_match = ['name']
index_id = ims_cli.create_index(
index_name,
content_type=IndexManagementService.COUCHDB_INDEX,
options=options,
datastore_name='fake',
view_name='fake/by_fake'
)
index_result = self.rr_cli.read(index_id)
self.assertIsInstance(index_result,CouchDBIndex)
self.assertTrue(index_result.name==index_name)
#======================================
# Clean up
#======================================
rr_cli.delete(index_id)
def test_read_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
index = ims_cli.read_index(index_id)
self.assertIsInstance(index,Index)
self.assertTrue(index.name==index_name)
rr_cli.delete(index_id)
def test_delete_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
ims_cli.delete_index(index_id)
with self.assertRaises(NotFound):
rr_cli.delete(index_id)
def test_update_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
index = ims_cli.read_index(index_id)
index.name = 'another'
ims_cli.update_index(index)
index = rr_cli.read(index_id)
self.assertTrue(index.name == 'another')
def test_find_indexes(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
#======================================
# Index Pool
#======================================
indexes = [
Index(name='first'),
Index(name='second'),
Index(name='third')
]
id_pool = list()
for index in indexes:
id_pool.append(rr_cli.create(index)[0])
index_id = ims_cli.find_indexes(index_name='second')
index = ims_cli.read_index(index_id)
self.assertTrue(index.name=='second')
#======================================
# Clean up
#======================================
for index_id in id_pool:
rr_cli.delete(index_id)
def test_create_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
with self.assertRaises(BadRequest):
ims_cli.create_collection('failing_collection')
resources = [ Resource(), Resource(), Resource() ]
resources = [ rr_cli.create(i)[0] for i in resources ]
collection_id = ims_cli.create_collection('working_collection',resources)
collection = rr_cli.read(collection_id)
collection_resources = ims_cli.list_collection_resources(collection_id, id_only=True)
self.assertTrue(set(collection_resources) == set(resources), '%s != %s' % (set(collection_resources) , set(resources)))
def test_read_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
collection = Collection(name='working_collection')
collection_id, _ = rr_cli.create(collection)
collection = ims_cli.read_collection(collection_id)
self.assertTrue(collection.name == 'working_collection')
def test_update_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
collection = Collection(name='useful_collection')
collection_id, _ = rr_cli.create(collection)
collection = rr_cli.read(collection_id)
collection.name = 'nub'
ims_cli.update_collection(collection)
collection = rr_cli.read(collection_id)
self.assertTrue(collection.name=='nub')
def test_delete_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
res = Resource()
res_id, rev = rr_cli.create(res)
collection_id = ims_cli.create_collection(name='test_collection', resources=[res_id])
ims_cli.delete_collection(collection_id)
with self.assertRaises(NotFound):
rr_cli.read(collection_id)
def test_list_collection_resources(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
#========================================
# Resource Pool
#========================================
resources = [ InformationResource(name='bean_counter'), InformationResource(name='lunar_rock'), InformationResource('aperature'), InformationResource('lemons') ]
resources = [ rr_cli.create(i)[0] for i in resources ]
collection = Collection(name='park_bench')
collection_id = ims_cli.create_collection(name='park_bench', resources=resources)
retval = ims_cli.list_collection_resources(collection_id, id_only=True)
retval.sort()
resources.sort()
self.assertTrue(retval == resources, '%s != %s' %(retval , resources))
def test_find_collection(self):
res_id, _ = self.rr_cli.create(Resource(name='test_res'))
collection_id = self.ims_cli.create_collection('test', [res_id])
retval = self.ims_cli.find_collection(collection_name='test')
self.assertTrue(retval[0] == collection_id)
retval = self.ims_cli.find_collection(resource_ids=[res_id])
self.assertTrue(retval[0] == collection_id)
class DiscoveryIntTest(IonIntegrationTestCase):
def setUp(self):
super(DiscoveryIntTest, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(
host=es_host,
port=es_port,
timeout=10
)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete,index)
IndexManagementService._es_call(es.index_delete,index)
def poll(self, tries, callback, *args, **kwargs):
'''
Polling wrapper for queries
Elasticsearch may not index and cache the changes right away so we may need
a couple of tries and a little time to go by before the results show.
'''
for i in xrange(tries):
retval = callback(*args, **kwargs)
if retval:
return retval
time.sleep(0.2)
return None
def test_traversal(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results = self.discovery.traverse(dp_id)
results.sort()
correct = [pd_id, transform_id]
correct.sort()
self.assertTrue(results == correct, '%s' % results)
def test_iterative_traversal(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results = self.discovery.iterative_traverse(dp_id)
results.sort()
correct = [transform_id]
self.assertTrue(results == correct)
results = self.discovery.iterative_traverse(dp_id, 1)
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@skipIf(not use_es, 'No ElasticSearch')
def test_view_crud(self):
view_id = self.discovery.create_view('big_view',fields=['name'])
catalog_id = self.discovery.list_catalogs(view_id)[0]
index_ids = self.catalog.list_indexes(catalog_id)
self.assertTrue(len(index_ids))
view = self.discovery.read_view(view_id)
self.assertIsInstance(view,View)
self.assertTrue(view.name == 'big_view')
view.name = 'not_so_big_view'
self.discovery.update_view(view)
view = self.discovery.read_view(view_id)
self.assertTrue(view.name == 'not_so_big_view')
self.discovery.delete_view(view_id)
with self.assertRaises(NotFound):
self.discovery.read_view(view_id)
def test_view_best_match(self):
#---------------------------------------------------------------
# Matches the best catalog available OR creates a new one
#---------------------------------------------------------------
catalog_id = self.catalog.create_catalog('dev', keywords=['name','model'])
view_id = self.discovery.create_view('exact_view', fields=['name','model'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids == [catalog_id])
view_id = self.discovery.create_view('another_view', fields=['name','model'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids == [catalog_id])
view_id = self.discovery.create_view('big_view', fields=['name'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids != [catalog_id])
@skipIf(not use_es, 'No ElasticSearch')
def test_basic_searching(self):
#- - - - - - - - - - - - - - - - -
# set up the fake resources
#- - - - - - - - - - - - - - - - -
instrument_pool = [
InstrumentDevice(name='sonobuoy1', firmware_version='1'),
InstrumentDevice(name='sonobuoy2', firmware_version='2'),
InstrumentDevice(name='sonobuoy3', firmware_version='3')
]
for instrument in instrument_pool:
self.rr.create(instrument)
view_id = self.discovery.create_view('devices', fields=['firmware_version'])
search_string = "search 'firmware_version' is '2' from '%s'"%view_id
results = self.poll(5, self.discovery.parse,search_string)
result = results[0]['_source']
self.assertIsInstance(result, InstrumentDevice)
self.assertTrue(result.name == 'sonobuoy2')
self.assertTrue(result.firmware_version == '2')
@skipIf(not use_es, 'No ElasticSearch')
def test_associative_searching(self):
dp_id,_ = self.rr.create(DataProduct('test_foo'))
ds_id,_ = self.rr.create(Dataset('test_bar', registered=True))
self.rr.create_association(subject=dp_id, object=ds_id, predicate='hasDataset')
search_string = "search 'type_' is 'Dataset' from 'resources_index' and belongs to '%s'" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(ds_id in results)
def test_iterative_associative_searching(self):
#--------------------------------------------------------------------------------
# Tests the ability to limit the iterations
#--------------------------------------------------------------------------------
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
search_string = "belongs to '%s' depth 1" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
correct = [transform_id]
self.assertTrue(results == correct, '%s' % results)
search_string = "belongs to '%s' depth 2" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@skipIf(not use_es, 'No ElasticSearch')
def test_ranged_value_searching(self):
discovery = self.discovery
rr = self.rr
view_id = discovery.create_view('bank_view', fields=['cash_balance'])
bank_id, _ = rr.create(BankAccount(name='broke', cash_balance=10))
search_string = "search 'cash_balance' values from 0 to 100 from '%s'" % view_id
results = self.poll(5, discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == bank_id)
bank_id, _ = rr.create(BankAccount(name='broke', cash_balance=90))
search_string = "search 'cash_balance' values from 80 from '%s'" % view_id
results = self.poll(5, discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == bank_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_collections_searching(self):
site_id, _ = self.rr.create(Site(name='black_mesa'))
view_id = self.discovery.create_view('big', fields=['name'])
# Add the site to a new collection
collection_id = self.ims.create_collection('resource_collection', [site_id])
search_string = "search 'name' is '*' from '%s' and in '%s'" %(view_id, collection_id)
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0] == site_id, '%s' % results)
@skipIf(not use_es, 'No ElasticSearch')
def test_search_by_name(self):
inst_dev = InstrumentDevice(name='test_dev',serial_number='ABC123')
dev_id, _ = self.rr.create(inst_dev)
self.discovery.create_view('devs',fields=['name','serial_number'])
search_string = "search 'serial_number' is 'abc*' from 'devs'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dev_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_search_by_name_index(self):
inst_dev = InstrumentDevice(name='test_dev',serial_number='ABC123')
dev_id, _ = self.rr.create(inst_dev)
search_string = "search 'serial_number' is 'abc*' from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dev_id)
bank_acc = BankAccount(name='blah', cash_balance=10)
res_id , _ = self.rr.create(bank_acc)
search_string = "search 'cash_balance' values from 0 to 100 from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == res_id)
#@skipIf(not use_es, 'No ElasticSearch')
@skip('Skip until time to refactor, data_format is removed from DataProduct resource')
def test_data_product_search(self):
# Create the dataproduct
dp = DataProduct(name='test_product')
dp.data_format.name = 'test_signal'
dp.data_format.description = 'test signal'
dp.data_format.character_set = 'utf8'
dp.data_format.nominal_sampling_rate_maximum = '44000'
dp.data_format.nominal_sampling_rate_minimum = '44000'
dp.CDM_data_type = 'basic'
dp_id, _ = self.rr.create(dp)
search_string = "search 'data_format.name' is 'test_signal' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'CDM_data_type' is 'basic' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'data_format.character_set' is 'utf8' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_events_search(self):
# Create a resource to force a new event
dp = DataProcess()
dp_id, rev = self.rr.create(dp)
search_string = "SEARCH 'origin' IS '%s' FROM 'events_index'" % dp_id
results = self.poll(9, self.discovery.parse,search_string)
origin_type = results[0]['_source'].origin_type
origin_id = results[0]['_source'].origin
self.assertTrue(origin_type == RT.DataProcess)
self.assertTrue(origin_id == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_geo_distance_search(self):
pd = PlatformDevice(name='test_dev')
pd_id, _ = self.rr.create(pd)
search_string = "search 'index_location' geo distance 20 km from lat 0 lon 0 from 'devices_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == pd_id)
self.assertTrue(results[0]['_source'].name == 'test_dev')
@skipIf(not use_es, 'No ElasticSearch')
def test_geo_bbox_search(self):
pd = PlatformDevice(name='test_dev')
pd.index_location.lat = 5
pd.index_location.lon = 5
pd_id, _ = self.rr.create(pd)
search_string = "search 'index_location' geo box top-left lat 10 lon 0 bottom-right lat 0 lon 10 from 'devices_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == pd_id)
self.assertTrue(results[0]['_source'].name == 'test_dev')
@skipIf(not use_es, 'No ElasticSearch')
def test_time_search(self):
today = date.today()
yesterday = today - timedelta(days=1)
tomorrow = today + timedelta(days=1)
data_product = DataProduct()
dp_id, _ = self.rr.create(data_product)
search_string = "search 'ts_created' time from '%s' to '%s' from 'data_products_index'" % (yesterday, tomorrow)
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results,'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'ts_created' time from '%s' from 'data_products_index'" % yesterday
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results,'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_user_search(self):
user = UserInfo()
user.name = 'test'
user.contact.phones.append('5551212')
user_id, _ = self.rr.create(user)
search_string = 'search "name" is "test" from "users_index"'
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == user_id)
self.assertTrue(results[0]['_source'].name == 'test')
search_string = 'search "contact.phones" is "5551212" from "users_index"'
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == user_id)
self.assertTrue(results[0]['_source'].name == 'test')
@skipIf(not use_es, 'No ElasticSearch')
def test_subobject_search(self):
contact = ContactInformation()
contact.email = '*****@*****.**'
contact.individual_name_family = 'Tester'
contact.individual_names_given = 'Intern'
dp = DataProduct(name='example')
dp.contacts.append(contact)
dp_id,_ = self.rr.create(dp)
#--------------------------------------------------------------------------------
# Example using the full field name
#--------------------------------------------------------------------------------
search_string = 'search "contacts.email" is "*****@*****.**" from "data_products"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
self.assertEquals(results[0]['_source'].name, 'example')
#--------------------------------------------------------------------------------
# Example using a sub-object's field name (ambiguous searching)
#--------------------------------------------------------------------------------
search_string = 'search "individual_names_given" is "Intern" from "data_products"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
self.assertEquals(results[0]['_source'].name, 'example')
@skipIf(not use_es, 'No ElasticSearch')
def test_descriptive_phrase_search(self):
dp = DataProduct(name='example', description='This is simply a description for this data product')
dp_id, _ = self.rr.create(dp)
search_string = 'search "description" like "description for" from "data_products"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
self.assertEquals(results[0]['_source'].name, 'example')
@skipIf(not use_es, 'No ElasticSearch')
def test_ownership_searching(self):
# Create two data products so that there is competition to the search, one is parsed
# (with conductivity as a parameter) and the other is raw
dp = DataProduct(name='example dataproduct')
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id)
tdom, sdom = time_series_domain()
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
dp_id = self.data_product_management.create_data_product(dp, stream_definition_id=stream_def_id, exchange_point='xp1')
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict')
stream_def_id = self.pubsub_management.create_stream_definition('ctd raw', parameter_dictionary_id=pdict_id)
dp = DataProduct(name='WRONG')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
self.data_product_management.create_data_product(dp, stream_definition_id=stream_def_id, exchange_point='xp1')
parameter_search = 'search "name" is "conductivity" from "resources_index"'
results = self.poll(9, self.discovery.parse, parameter_search)
param_id = results[0]['_id']
data_product_search = 'search "name" is "*" from "data_products_index" and has "%s"' % param_id
results = self.poll(9, self.discovery.parse, data_product_search)
print results
self.assertEquals(results[0], dp_id)
class DiscoveryIntTest(IonIntegrationTestCase):
def setUp(self):
super(DiscoveryIntTest, self).setUp()
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
if use_es:
self.es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
self.es_port = CFG.get_safe('server.elasticsearch.port', '9200')
CFG.server.elasticsearch.shards = 1
CFG.server.elasticsearch.replicas = 0
CFG.server.elasticsearch.river_shards = 1
CFG.server.elasticsearch.river_replicas = 0
self.es = ep.ElasticSearch(
host=self.es_host,
port=self.es_port,
timeout=10,
verbose=True
)
op = DotDict(CFG)
op.op = 'clean_bootstrap'
self.container.spawn_process('index_bootstrap','ion.processes.bootstrap.index_bootstrap','IndexBootStrap', op)
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(
host=es_host,
port=es_port,
timeout=10
)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete,index)
IndexManagementService._es_call(es.index_delete,index)
def poll(self, tries, callback, *args, **kwargs):
'''
Polling wrapper for queries
Elasticsearch may not index and cache the changes right away so we may need
a couple of tries and a little time to go by before the results show.
'''
for i in xrange(tries):
retval = callback(*args, **kwargs)
if retval:
return retval
time.sleep(0.2)
return None
def test_traversal(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results = self.discovery.traverse(dp_id)
results.sort()
correct = [pd_id, transform_id]
correct.sort()
self.assertTrue(results == correct, '%s' % results)
def test_iterative_traversal(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results = self.discovery.iterative_traverse(dp_id)
results.sort()
correct = [transform_id]
self.assertTrue(results == correct)
results = self.discovery.iterative_traverse(dp_id, 1)
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@skipIf(not use_es, 'No ElasticSearch')
def test_view_crud(self):
view_id = self.discovery.create_view('big_view',fields=['name'])
catalog_id = self.discovery.list_catalogs(view_id)[0]
index_ids = self.catalog.list_indexes(catalog_id)
self.assertTrue(len(index_ids))
view = self.discovery.read_view(view_id)
self.assertIsInstance(view,View)
self.assertTrue(view.name == 'big_view')
view.name = 'not_so_big_view'
self.discovery.update_view(view)
view = self.discovery.read_view(view_id)
self.assertTrue(view.name == 'not_so_big_view')
self.discovery.delete_view(view_id)
with self.assertRaises(NotFound):
self.discovery.read_view(view_id)
def test_view_best_match(self):
#---------------------------------------------------------------
# Matches the best catalog available OR creates a new one
#---------------------------------------------------------------
catalog_id = self.catalog.create_catalog('dev', keywords=['name','model'])
view_id = self.discovery.create_view('exact_view', fields=['name','model'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids == [catalog_id])
view_id = self.discovery.create_view('another_view', fields=['name','model'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids == [catalog_id])
view_id = self.discovery.create_view('big_view', fields=['name'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids != [catalog_id])
@skipIf(not use_es, 'No ElasticSearch')
def test_basic_searching(self):
#- - - - - - - - - - - - - - - - -
# set up the fake resources
#- - - - - - - - - - - - - - - - -
instrument_pool = [
InstrumentDevice(name='sonobuoy1', hardware_version='1'),
InstrumentDevice(name='sonobuoy2', hardware_version='2'),
InstrumentDevice(name='sonobuoy3', hardware_version='3')
]
for instrument in instrument_pool:
self.rr.create(instrument)
view_id = self.discovery.create_view('devices', fields=['hardware_version'])
search_string = "search 'hardware_version' is '2' from '%s'"%view_id
results = self.poll(5, self.discovery.parse,search_string)
result = results[0]['_source']
self.assertIsInstance(result, InstrumentDevice)
self.assertTrue(result.name == 'sonobuoy2')
self.assertTrue(result.hardware_version == '2')
@skipIf(not use_es, 'No ElasticSearch')
def test_associative_searching(self):
view_id = self.discovery.create_view('devices', fields=['model'])
site_id,_ = self.rr.create(Site('my_site'))
pd_id, _ = self.rr.create(PlatformDevice('my_device', model='abc123'))
self.rr.create_association(subject=site_id, object=pd_id, predicate=PRED.hasDevice)
search_string = "search 'model' is 'abc*' from '%s' and belongs to '%s'"%(view_id, site_id)
results = self.poll(5, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(pd_id in results)
def test_iterative_associative_searching(self):
#--------------------------------------------------------------------------------
# Tests the ability to limit the iterations
#--------------------------------------------------------------------------------
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
search_string = "belongs to '%s' depth 1" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
correct = [transform_id]
self.assertTrue(results == correct, '%s' % results)
search_string = "belongs to '%s' depth 2" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@skipIf(not use_es, 'No ElasticSearch')
def test_ranged_value_searching(self):
discovery = self.discovery
rr = self.rr
view_id = discovery.create_view('bank_view', fields=['cash_balance'])
bank_id, _ = rr.create(BankAccount(name='broke', cash_balance=10))
search_string = "search 'cash_balance' values from 0 to 100 from '%s'" % view_id
results = self.poll(5, discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == bank_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_collections_searching(self):
site_id, _ = self.rr.create(Site(name='black_mesa'))
view_id = self.discovery.create_view('big', fields=['name'])
# Add the site to a new collection
collection_id = self.ims.create_collection('resource_collection', [site_id])
search_string = "search 'name' is '*' from '%s' and in '%s'" %(view_id, collection_id)
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0] == site_id, '%s' % results)
@skipIf(not use_es, 'No ElasticSearch')
def test_search_by_name(self):
inst_dev = InstrumentDevice(name='test_dev',serial_number='ABC123')
dev_id, _ = self.rr.create(inst_dev)
self.discovery.create_view('devs',fields=['name','serial_number'])
search_string = "search 'serial_number' is 'abc*' from 'devs'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dev_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_search_by_name_index(self):
inst_dev = InstrumentDevice(name='test_dev',serial_number='ABC123')
dev_id, _ = self.rr.create(inst_dev)
search_string = "search 'serial_number' is 'abc*' from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dev_id)
bank_acc = BankAccount(name='blah', cash_balance=10)
res_id , _ = self.rr.create(bank_acc)
search_string = "search 'cash_balance' values from 0 to 100 from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == res_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_data_product_search(self):
# Create the dataproduct
dp = DataProduct(name='test_product')
dp.data_format.name = 'test_signal'
dp.data_format.description = 'test signal'
dp.data_format.character_set = 'utf8'
dp.data_format.nominal_sampling_rate_maximum = '44000'
dp.data_format.nominal_sampling_rate_minimum = '44000'
dp.data_product_level = 'basic'
dp_id, _ = self.rr.create(dp)
search_string = "search 'data_format.name' is 'test_signal' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'data_product_level' is 'basic' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'data_format.character_set' is 'utf8' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_events_search(self):
# Create a resource to force a new event
dp = DataProcess()
dp_id, rev = self.rr.create(dp)
search_string = "SEARCH 'origin' IS '%s' FROM 'events_index'" % dp_id
results = self.poll(9, self.discovery.parse,search_string)
origin_type = results[0]['_source'].origin_type
origin_id = results[0]['_source'].origin
self.assertTrue(origin_type == RT.DataProcess)
self.assertTrue(origin_id == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_geo_distance_search(self):
pd = PlatformDevice(name='test_dev')
pd_id, _ = self.rr.create(pd)
search_string = "search 'nominal_location' geo distance 20 km from lat 0 lon 0 from 'devices_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == pd_id)
self.assertTrue(results[0]['_source'].name == 'test_dev')
@skipIf(not use_es, 'No ElasticSearch')
def test_geo_bbox_search(self):
pd = PlatformDevice(name='test_dev')
pd.nominal_location.lat = 5
pd.nominal_location.lon = 5
pd_id, _ = self.rr.create(pd)
search_string = "search 'nominal_location' geo box top-left lat 10 lon 0 bottom-right lat 0 lon 10 from 'devices_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == pd_id)
self.assertTrue(results[0]['_source'].name == 'test_dev')
@skipIf(not use_es, 'No ElasticSearch')
def test_user_search(self):
user = UserInfo()
user.name = 'test'
user.contact.phone = '5551212'
user_id, _ = self.rr.create(user)
search_string = 'search "name" is "test" from "users_index"'
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == user_id)
self.assertTrue(results[0]['_source'].name == 'test')
search_string = 'search "contact.phone" is "5551212" from "users_index"'
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == user_id)
self.assertTrue(results[0]['_source'].name == 'test')
def index_bootstrap(self):
'''
Creates the initial set of desired indexes based on a standard definition
'''
#=======================================================================================
# Create the _river index based on the cluster configurations
#=======================================================================================
IndexManagementService._es_call(
self.es.index_create,
'_river',
number_of_shards = self.river_shards,
number_of_replicas = self.river_replicas
)
filters = {
'_id' : '_design/filters',
'filters' : {
}
}
#=======================================================================================
# For each of the resource types in the list of values for each standard index,
# create a mapping and a context type in ElasticSearch based on the searchable fields.
#=======================================================================================
for k,v in STD_INDEXES.iteritems():
response = IndexManagementService._es_call(
self.es.index_create,
k,
number_of_shards=self.index_shards,
number_of_replicas=self.index_replicas
)
IndexManagementService._check_response(response)
body = 'function(doc, req) { switch(doc.type_) { default: return false; }}'
for res in v:
body = re.sub(r'default:', 'case "%s": return true; default:' % res, body)
mappings = self.es_mapping(res)
response = IndexManagementService._es_call(self.es.raw,'%s/%s/_mapping' %(k,res), 'POST', mappings)
IndexManagementService._check_response(response)
filters['filters'][k] = body
#=======================================================================================
# Get an instance of the datastore instance used to create the CouchDB filters
# in support of the ElasticSearch river's filter
# - Allows us to filter based on resource type
#=======================================================================================
cc = self.container
db = cc.datastore_manager.get_datastore('resources')
datastore_name = db.datastore_name
db = db.server[datastore_name]
db.create(filters)
#--------------------------------------------------------------------------------
# Create the river connection between CouchDB and ElasticSearch
#--------------------------------------------------------------------------------
for k,v in STD_INDEXES.iteritems():
response = IndexManagementService._es_call(self.es.river_couchdb_create,
index_name = k,
couchdb_db = datastore_name,
couchdb_host = CFG.server.couchdb.host,
couchdb_port = CFG.server.couchdb.port,
couchdb_user = CFG.server.couchdb.username,
couchdb_password = CFG.server.couchdb.password,
couchdb_filter = 'filters/%s' % k,
script= ELASTICSEARCH_CONTEXT_SCRIPT
)
IndexManagementService._check_response(response)
#=======================================================================================
# Create and map the edge indexes
#=======================================================================================
#--------------------------------------------------------------------------------
# Resources Index
#--------------------------------------------------------------------------------
response = IndexManagementService._es_call(self.es.index_create,'%s_resources_index' % self.sysname,
number_of_shards=self.index_shards,
number_of_replicas=self.index_replicas
)
IndexManagementService._check_response(response)
for t in RT.values():
mappings = self.es_mapping(t)
response = IndexManagementService._es_call(self.es.raw,'%s_resources_index/%s/_mapping' % (self.sysname,t), 'POST', mappings)
IndexManagementService._check_response(response)
response = IndexManagementService._es_call(self.es.river_couchdb_create,
index_name = '%s_resources_index' % self.sysname,
couchdb_db = datastore_name,
couchdb_host = CFG.server.couchdb.host,
couchdb_port = CFG.server.couchdb.port,
couchdb_user = CFG.server.couchdb.username,
couchdb_password = CFG.server.couchdb.password,
script= ELASTICSEARCH_CONTEXT_SCRIPT
)
IndexManagementService._check_response(response)
#--------------------------------------------------------------------------------
# Events Index
#--------------------------------------------------------------------------------
response = IndexManagementService._es_call(self.es.index_create,'%s_events_index' % self.sysname,
number_of_shards=self.index_shards,
number_of_replicas=self.index_replicas
)
IndexManagementService._check_response(response)
for event in get_events():
mappings = self.es_mapping(event)
response = IndexManagementService._es_call(self.es.raw, '%s_events_index/%s/_mapping' % (self.sysname, event), 'POST', mappings)
IndexManagementService._check_response(response)
response = IndexManagementService._es_call(self.es.river_couchdb_create,
index_name = '%s_events_index' % self.sysname,
couchdb_db = '%s_events' % self.sysname,
couchdb_host = CFG.server.couchdb.host,
couchdb_port = CFG.server.couchdb.port,
couchdb_user = CFG.server.couchdb.username,
couchdb_password = CFG.server.couchdb.password,
script= ELASTICSEARCH_CONTEXT_SCRIPT
)
IndexManagementService._check_response(response)
#=======================================================================================
# Construct the resources
#=======================================================================================
ims_cli = IndexManagementServiceClient()
#--------------------------------------------------------------------------------
# Standard Indexes
#--------------------------------------------------------------------------------
for index,resources in STD_INDEXES.iteritems():
ims_cli.create_index(
name=index,
description='%s ElasticSearch Index Resource' % index,
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=self.attr_mapping(resources)
)
#--------------------------------------------------------------------------------
# CouchDB Indexes
#--------------------------------------------------------------------------------
for index,datastore in COUCHDB_INDEXES.iteritems():
ims_cli.create_index(
name=index,
description='%s CouchDB Index Resource' % index,
content_type=IndexManagementService.COUCHDB_INDEX,
datastore_name=datastore
)
#--------------------------------------------------------------------------------
# Edge Indexes
#--------------------------------------------------------------------------------
ims_cli.create_index(
name='%s_resources_index' % self.sysname,
description='Resources Index',
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=self.attr_mapping(RT.keys())
)
ims_cli.create_index(
name='%s_events_index' % self.sysname,
description='Events Index',
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=self.attr_mapping(get_events())
)
class IndexManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.ims_cli = IndexManagementServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
self.index_name = 'test_index'
def test_create_datastore_index(self):
index_name = self.index_name
ims_cli = self.ims_cli
rr_cli = self.rr_cli
options = SearchOptions()
options.attribute_match = ['test_field']
index_id = ims_cli.create_index(
name=index_name,
content_type=IndexManagementService.DATASTORE_INDEX,
options=options)
index_result = self.rr_cli.read(index_id)
self.assertIsInstance(index_result, ElasticSearchIndex)
self.assertTrue(index_result.name == index_name)
#======================================
# Clean up
#======================================
rr_cli.delete(index_id)
def test_read_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
index = ims_cli.read_index(index_id)
self.assertIsInstance(index, Index)
self.assertTrue(index.name == index_name)
rr_cli.delete(index_id)
def test_delete_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
ims_cli.delete_index(index_id)
with self.assertRaises(NotFound):
rr_cli.delete(index_id)
def test_update_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
index = ims_cli.read_index(index_id)
index.name = 'another'
ims_cli.update_index(index)
index = rr_cli.read(index_id)
self.assertTrue(index.name == 'another')
def test_find_indexes(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
#======================================
# Index Pool
#======================================
indexes = [
Index(name='first'),
Index(name='second'),
Index(name='third')
]
id_pool = list()
for index in indexes:
id_pool.append(rr_cli.create(index)[0])
index_id = ims_cli.find_indexes(index_name='second')
index = ims_cli.read_index(index_id)
self.assertTrue(index.name == 'second')
#======================================
# Clean up
#======================================
for index_id in id_pool:
rr_cli.delete(index_id)
def test_create_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
with self.assertRaises(BadRequest):
ims_cli.create_collection('failing_collection')
resources = [Resource(), Resource(), Resource()]
resources = [rr_cli.create(i)[0] for i in resources]
collection_id = ims_cli.create_collection('working_collection',
resources)
collection = rr_cli.read(collection_id)
collection_resources = ims_cli.list_collection_resources(collection_id,
id_only=True)
self.assertTrue(
set(collection_resources) == set(resources),
'%s != %s' % (set(collection_resources), set(resources)))
def test_read_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
collection = Collection(name='working_collection')
collection_id, _ = rr_cli.create(collection)
collection = ims_cli.read_collection(collection_id)
self.assertTrue(collection.name == 'working_collection')
def test_update_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
collection = Collection(name='useful_collection')
collection_id, _ = rr_cli.create(collection)
collection = rr_cli.read(collection_id)
collection.name = 'nub'
ims_cli.update_collection(collection)
collection = rr_cli.read(collection_id)
self.assertTrue(collection.name == 'nub')
def test_delete_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
res = Resource()
res_id, rev = rr_cli.create(res)
collection_id = ims_cli.create_collection(name='test_collection',
resources=[res_id])
ims_cli.delete_collection(collection_id)
with self.assertRaises(NotFound):
rr_cli.read(collection_id)
def test_list_collection_resources(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
#========================================
# Resource Pool
#========================================
resources = [
InformationResource(name='bean_counter'),
InformationResource(name='lunar_rock'),
InformationResource('aperature'),
InformationResource('lemons')
]
resources = [rr_cli.create(i)[0] for i in resources]
collection = Collection(name='park_bench')
collection_id = ims_cli.create_collection(name='park_bench',
resources=resources)
retval = ims_cli.list_collection_resources(collection_id, id_only=True)
retval.sort()
resources.sort()
self.assertTrue(retval == resources, '%s != %s' % (retval, resources))
def test_find_collection(self):
res_id, _ = self.rr_cli.create(Resource(name='test_res'))
collection_id = self.ims_cli.create_collection('test', [res_id])
retval = self.ims_cli.find_collection(collection_name='test')
self.assertTrue(retval[0] == collection_id)
retval = self.ims_cli.find_collection(resource_ids=[res_id])
self.assertTrue(retval[0] == collection_id)
def index_bootstrap(self):
'''
Creates the initial set of desired indexes based on a standard definition
'''
#=======================================================================================
# Create the _river index based on the cluster configurations
#=======================================================================================
IndexManagementService._es_call(self.es.index_create,
'_river',
number_of_shards=self.river_shards,
number_of_replicas=self.river_replicas)
filters = {'_id': '_design/filters', 'filters': {}}
#=======================================================================================
# For each of the resource types in the list of values for each standard index,
# create a mapping and a context type in ElasticSearch based on the searchable fields.
#=======================================================================================
for k, v in STD_INDEXES.iteritems():
response = IndexManagementService._es_call(
self.es.index_create,
k,
number_of_shards=self.index_shards,
number_of_replicas=self.index_replicas)
IndexManagementService._check_response(response)
body = 'function(doc, req) { switch(doc.type_) { default: return false; }}'
for res in v:
body = re.sub(r'default:',
'case "%s": return true; default:' % res, body)
mappings = self.es_mapping(res)
response = IndexManagementService._es_call(
self.es.raw, '%s/%s/_mapping' % (k, res), 'POST', mappings)
IndexManagementService._check_response(response)
filters['filters'][k] = body
#=======================================================================================
# Get an instance of the datastore instance used to create the CouchDB filters
# in support of the ElasticSearch river's filter
# - Allows us to filter based on resource type
#=======================================================================================
cc = self.container
db = cc.datastore_manager.get_datastore('resources')
datastore_name = db.datastore_name
db = db.server[datastore_name]
db.create(filters)
#--------------------------------------------------------------------------------
# Create the river connection between CouchDB and ElasticSearch
#--------------------------------------------------------------------------------
for k, v in STD_INDEXES.iteritems():
response = IndexManagementService._es_call(
self.es.river_couchdb_create,
index_name=k,
couchdb_db=datastore_name,
couchdb_host=CFG.server.couchdb.host,
couchdb_port=CFG.server.couchdb.port,
couchdb_user=CFG.server.couchdb.username,
couchdb_password=CFG.server.couchdb.password,
couchdb_filter='filters/%s' % k,
script=ELASTICSEARCH_CONTEXT_SCRIPT)
IndexManagementService._check_response(response)
#=======================================================================================
# Create and map the edge indexes
#=======================================================================================
#--------------------------------------------------------------------------------
# Resources Index
#--------------------------------------------------------------------------------
response = IndexManagementService._es_call(
self.es.index_create,
'%s_resources_index' % self.sysname,
number_of_shards=self.index_shards,
number_of_replicas=self.index_replicas)
IndexManagementService._check_response(response)
for t in RT.values():
mappings = self.es_mapping(t)
response = IndexManagementService._es_call(
self.es.raw,
'%s_resources_index/%s/_mapping' % (self.sysname, t), 'POST',
mappings)
IndexManagementService._check_response(response)
response = IndexManagementService._es_call(
self.es.river_couchdb_create,
index_name='%s_resources_index' % self.sysname,
couchdb_db=datastore_name,
couchdb_host=CFG.server.couchdb.host,
couchdb_port=CFG.server.couchdb.port,
couchdb_user=CFG.server.couchdb.username,
couchdb_password=CFG.server.couchdb.password,
script=ELASTICSEARCH_CONTEXT_SCRIPT)
IndexManagementService._check_response(response)
#--------------------------------------------------------------------------------
# Events Index
#--------------------------------------------------------------------------------
response = IndexManagementService._es_call(
self.es.index_create,
'%s_events_index' % self.sysname,
number_of_shards=self.index_shards,
number_of_replicas=self.index_replicas)
IndexManagementService._check_response(response)
for event in get_events():
mappings = self.es_mapping(event)
response = IndexManagementService._es_call(
self.es.raw,
'%s_events_index/%s/_mapping' % (self.sysname, event), 'POST',
mappings)
IndexManagementService._check_response(response)
response = IndexManagementService._es_call(
self.es.river_couchdb_create,
index_name='%s_events_index' % self.sysname,
couchdb_db='%s_events' % self.sysname,
couchdb_host=CFG.server.couchdb.host,
couchdb_port=CFG.server.couchdb.port,
couchdb_user=CFG.server.couchdb.username,
couchdb_password=CFG.server.couchdb.password,
script=ELASTICSEARCH_CONTEXT_SCRIPT)
IndexManagementService._check_response(response)
#=======================================================================================
# Construct the resources
#=======================================================================================
ims_cli = IndexManagementServiceClient()
#--------------------------------------------------------------------------------
# Standard Indexes
#--------------------------------------------------------------------------------
for index, resources in STD_INDEXES.iteritems():
ims_cli.create_index(
name=index,
description='%s ElasticSearch Index Resource' % index,
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=self.attr_mapping(resources))
#--------------------------------------------------------------------------------
# CouchDB Indexes
#--------------------------------------------------------------------------------
for index, datastore in COUCHDB_INDEXES.iteritems():
ims_cli.create_index(
name=index,
description='%s CouchDB Index Resource' % index,
content_type=IndexManagementService.COUCHDB_INDEX,
datastore_name=datastore)
#--------------------------------------------------------------------------------
# Edge Indexes
#--------------------------------------------------------------------------------
ims_cli.create_index(
name='%s_resources_index' % self.sysname,
description='Resources Index',
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=self.attr_mapping(RT.keys()))
ims_cli.create_index(
name='%s_events_index' % self.sysname,
description='Events Index',
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=self.attr_mapping(get_events()))
class DiscoveryIntTest(IonIntegrationTestCase):
def setUp(self):
super(DiscoveryIntTest, self).setUp()
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
if use_es:
self.es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
self.es_port = CFG.get_safe('server.elasticsearch.port', '9200')
CFG.server.elasticsearch.shards = 1
CFG.server.elasticsearch.replicas = 0
CFG.server.elasticsearch.river_shards = 1
CFG.server.elasticsearch.river_replicas = 0
self.es = ep.ElasticSearch(
host=self.es_host,
port=self.es_port,
timeout=10,
verbose=True
)
op = DotDict(CFG)
op.op = 'clean_bootstrap'
self.container.spawn_process('index_bootstrap','ion.processes.bootstrap.index_bootstrap','IndexBootStrap', op)
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(
host=es_host,
port=es_port,
timeout=10
)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete,index)
IndexManagementService._es_call(es.index_delete,index)
def poll(self, tries, callback, *args, **kwargs):
'''
Polling wrapper for queries
Elasticsearch may not index and cache the changes right away so we may need
a couple of tries and a little time to go by before the results show.
'''
for i in xrange(tries):
retval = callback(*args, **kwargs)
if retval:
return retval
time.sleep(0.2)
return None
def test_traversal(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results = self.discovery.traverse(dp_id)
results.sort()
correct = [pd_id, transform_id]
correct.sort()
self.assertTrue(results == correct, '%s' % results)
def test_iterative_traversal(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results = self.discovery.iterative_traverse(dp_id)
results.sort()
correct = [transform_id]
self.assertTrue(results == correct)
results = self.discovery.iterative_traverse(dp_id, 1)
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@skipIf(not use_es, 'No ElasticSearch')
def test_view_crud(self):
view_id = self.discovery.create_view('big_view',fields=['name'])
catalog_id = self.discovery.list_catalogs(view_id)[0]
index_ids = self.catalog.list_indexes(catalog_id)
self.assertTrue(len(index_ids))
view = self.discovery.read_view(view_id)
self.assertIsInstance(view,View)
self.assertTrue(view.name == 'big_view')
view.name = 'not_so_big_view'
self.discovery.update_view(view)
view = self.discovery.read_view(view_id)
self.assertTrue(view.name == 'not_so_big_view')
self.discovery.delete_view(view_id)
with self.assertRaises(NotFound):
self.discovery.read_view(view_id)
def test_view_best_match(self):
#---------------------------------------------------------------
# Matches the best catalog available OR creates a new one
#---------------------------------------------------------------
catalog_id = self.catalog.create_catalog('dev', keywords=['name','model'])
view_id = self.discovery.create_view('exact_view', fields=['name','model'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids == [catalog_id])
view_id = self.discovery.create_view('another_view', fields=['name','model'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids == [catalog_id])
view_id = self.discovery.create_view('big_view', fields=['name'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids != [catalog_id])
@skipIf(not use_es, 'No ElasticSearch')
def test_basic_searching(self):
#- - - - - - - - - - - - - - - - -
# set up the fake resources
#- - - - - - - - - - - - - - - - -
instrument_pool = [
InstrumentDevice(name='sonobuoy1', hardware_version='1'),
InstrumentDevice(name='sonobuoy2', hardware_version='2'),
InstrumentDevice(name='sonobuoy3', hardware_version='3')
]
for instrument in instrument_pool:
self.rr.create(instrument)
view_id = self.discovery.create_view('devices', fields=['hardware_version'])
search_string = "search 'hardware_version' is '2' from '%s'"%view_id
results = self.poll(5, self.discovery.parse,search_string)
result = results[0]['_source']
self.assertIsInstance(result, InstrumentDevice)
self.assertTrue(result.name == 'sonobuoy2')
self.assertTrue(result.hardware_version == '2')
@skipIf(not use_es, 'No ElasticSearch')
def test_associative_searching(self):
view_id = self.discovery.create_view('devices', fields=['model'])
site_id,_ = self.rr.create(Site('my_site'))
pd_id, _ = self.rr.create(PlatformDevice('my_device', model='abc123'))
self.rr.create_association(subject=site_id, object=pd_id, predicate=PRED.hasDevice)
search_string = "search 'model' is 'abc*' from '%s' and belongs to '%s'"%(view_id, site_id)
results = self.poll(5, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(pd_id in results)
def test_iterative_associative_searching(self):
#--------------------------------------------------------------------------------
# Tests the ability to limit the iterations
#--------------------------------------------------------------------------------
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
search_string = "belongs to '%s' depth 1" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
correct = [transform_id]
self.assertTrue(results == correct, '%s' % results)
search_string = "belongs to '%s' depth 2" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@skipIf(not use_es, 'No ElasticSearch')
def test_ranged_value_searching(self):
discovery = self.discovery
rr = self.rr
view_id = discovery.create_view('bank_view', fields=['cash_balance'])
bank_id, _ = rr.create(BankAccount(name='broke', cash_balance=10))
search_string = "search 'cash_balance' values from 0 to 100 from '%s'" % view_id
results = self.poll(5, discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == bank_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_collections_searching(self):
site_id, _ = self.rr.create(Site(name='black_mesa'))
view_id = self.discovery.create_view('big', fields=['name'])
# Add the site to a new collection
collection_id = self.ims.create_collection('resource_collection', [site_id])
search_string = "search 'name' is '*' from '%s' and in '%s'" %(view_id, collection_id)
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0] == site_id, '%s' % results)
@skipIf(not use_es, 'No ElasticSearch')
def test_search_by_name(self):
inst_dev = InstrumentDevice(name='test_dev',serial_number='ABC123')
dev_id, _ = self.rr.create(inst_dev)
self.discovery.create_view('devs',fields=['name','serial_number'])
search_string = "search 'serial_number' is 'abc*' from 'devs'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dev_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_search_by_name_index(self):
inst_dev = InstrumentDevice(name='test_dev',serial_number='ABC123')
dev_id, _ = self.rr.create(inst_dev)
search_string = "search 'serial_number' is 'abc*' from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dev_id)
bank_acc = BankAccount(name='blah', cash_balance=10)
res_id , _ = self.rr.create(bank_acc)
search_string = "search 'cash_balance' values from 0 to 100 from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == res_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_data_product_search(self):
# Create the dataproduct
dp = DataProduct(name='test_product')
dp.data_format.name = 'test_signal'
dp.data_format.description = 'test signal'
dp.data_format.character_set = 'utf8'
dp.data_format.nominal_sampling_rate_maximum = '44000'
dp.data_format.nominal_sampling_rate_minimum = '44000'
dp.data_product_level = 'basic'
dp_id, _ = self.rr.create(dp)
search_string = "search 'data_format.name' is 'test_signal' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'data_product_level' is 'basic' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'data_format.character_set' is 'utf8' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_events_search(self):
# Create a resource to force a new event
dp = DataProcess()
dp_id, rev = self.rr.create(dp)
search_string = "SEARCH 'origin' IS '%s' FROM 'events_index'" % dp_id
results = self.poll(9, self.discovery.parse,search_string)
origin_type = results[0]['_source'].origin_type
origin_id = results[0]['_source'].origin
self.assertTrue(origin_type == RT.DataProcess)
self.assertTrue(origin_id == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_geo_distance_search(self):
pd = PlatformDevice(name='test_dev')
pd_id, _ = self.rr.create(pd)
search_string = "search 'nominal_location' geo distance 20 km from lat 0 lon 0 from 'devices_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == pd_id)
self.assertTrue(results[0]['_source'].name == 'test_dev')
@skipIf(not use_es, 'No ElasticSearch')
def test_geo_bbox_search(self):
pd = PlatformDevice(name='test_dev')
pd.nominal_location.lat = 5
pd.nominal_location.lon = 5
pd_id, _ = self.rr.create(pd)
search_string = "search 'nominal_location' geo box top-left lat 10 lon 0 bottom-right lat 0 lon 10 from 'devices_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == pd_id)
self.assertTrue(results[0]['_source'].name == 'test_dev')
class DiscoveryIntTest(IonIntegrationTestCase):
def setUp(self):
raise SkipTest("Not yet ported to Postgres")
super(DiscoveryIntTest, self).setUp()
config = DotDict()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
def test_geo_distance_search(self):
pd = PlatformDevice(name='test_dev')
pd_id, _ = self.rr.create(pd)
search_string = "search 'index_location' geo distance 20 km from lat 0 lon 0 from 'devices_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == pd_id)
self.assertTrue(results[0]['_source'].name == 'test_dev')
def test_ranged_value_searching(self):
discovery = self.discovery
rr = self.rr
view_id = discovery.create_view('bank_view', fields=['cash_balance'])
bank_id, _ = rr.create(BankAccount(name='broke', cash_balance=10))
search_string = "search 'cash_balance' values from 0 to 100 from '%s'" % view_id
results = self.poll(5, discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == bank_id)
bank_id, _ = rr.create(BankAccount(name='broke', cash_balance=90))
search_string = "search 'cash_balance' values from 80 from '%s'" % view_id
results = self.poll(5, discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == bank_id)
def test_collections_searching(self):
site_id, _ = self.rr.create(Site(name='black_mesa'))
view_id = self.discovery.create_view('big', fields=['name'])
# Add the site to a new collection
collection_id = self.ims.create_collection('resource_collection', [site_id])
search_string = "search 'name' is '*' from '%s' and in '%s'" %(view_id, collection_id)
results = self.poll(9, self.discovery.parse,search_string,id_only=True)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0] == site_id, '%s' % results)
def test_search_by_name_index(self):
inst_dev = InstrumentDevice(name='test_dev',serial_number='ABC123')
dev_id, _ = self.rr.create(inst_dev)
search_string = "search 'serial_number' is 'abc*' from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dev_id)
bank_acc = BankAccount(name='blah', cash_balance=10)
res_id , _ = self.rr.create(bank_acc)
search_string = "search 'cash_balance' values from 0 to 100 from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == res_id)
def test_data_product_search(self):
# Create the dataproduct
dp = DataProduct(name='test_product')
dp.data_format.name = 'test_signal'
dp.data_format.description = 'test signal'
dp.data_format.character_set = 'utf8'
dp.data_format.nominal_sampling_rate_maximum = '44000'
dp.data_format.nominal_sampling_rate_minimum = '44000'
dp.CDM_data_type = 'basic'
dp_id, _ = self.rr.create(dp)
search_string = "search 'data_format.name' is 'test_signal' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'CDM_data_type' is 'basic' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'data_format.character_set' is 'utf8' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
def test_events_search(self):
# Create a resource to force a new event
dp = DataProcess()
dp_id, rev = self.rr.create(dp)
search_string = "SEARCH 'origin' IS '%s' FROM 'events_index'" % dp_id
results = self.poll(9, self.discovery.parse,search_string)
origin_type = results[0]['_source'].origin_type
origin_id = results[0]['_source'].origin
self.assertTrue(origin_type == RT.DataProcess)
self.assertTrue(origin_id == dp_id)
def test_time_search(self):
today = date.today()
past = today - timedelta(days=2)
future = today + timedelta(days=2)
data_product = DataProduct()
dp_id, _ = self.rr.create(data_product)
search_string = "search 'type_' is 'DataProduct' from 'data_products_index' and search 'ts_created' time from '%s' to '%s' from 'data_products_index'" % (past, future)
results = self.poll(9, self.discovery.parse,search_string,id_only=True)
self.assertIsNotNone(results,'Results not found')
self.assertIn(dp_id, results)
search_string = "search 'type_' is 'DataProduct' from 'data_products_index' and search 'ts_created' time from '%s' from 'data_products_index'" % past
results = self.poll(9, self.discovery.parse,search_string,id_only=True)
self.assertIsNotNone(results,'Results not found')
self.assertIn(dp_id, results)
def test_user_search(self):
user = UserInfo()
user.name = 'test'
user.contact.phones.append('5551212')
user_id, _ = self.rr.create(user)
search_string = 'search "name" is "test" from "users_index"'
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == user_id)
self.assertTrue(results[0]['_source'].name == 'test')
search_string = 'search "contact.phones" is "5551212" from "users_index"'
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == user_id)
self.assertTrue(results[0]['_source'].name == 'test')
def test_subobject_search(self):
contact = ContactInformation()
contact.email = '*****@*****.**'
contact.individual_name_family = 'Tester'
contact.individual_names_given = 'Intern'
dp = DataProduct(name='example')
dp.contacts.append(contact)
dp_id,_ = self.rr.create(dp)
#--------------------------------------------------------------------------------
# Example using the full field name
#--------------------------------------------------------------------------------
search_string = 'search "contacts.email" is "*****@*****.**" from "data_products"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
self.assertEquals(results[0]['_source'].name, 'example')
#--------------------------------------------------------------------------------
# Example using a sub-object's field name (ambiguous searching)
#--------------------------------------------------------------------------------
search_string = 'search "individual_names_given" is "Intern" from "data_products"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
self.assertEquals(results[0]['_source'].name, 'example')
def test_descriptive_phrase_search(self):
dp = DataProduct(name='example', description='This is simply a description for this data product')
dp_id, _ = self.rr.create(dp)
search_string = 'search "description" like "description for" from "data_products"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
self.assertEquals(results[0]['_source'].name, 'example')
def test_match_search(self):
dp = DataProduct(name='example', description='This is simply a description for this data product')
dp_id, _ = self.rr.create(dp)
search_string = 'search "description" match "this data product" from "data_products"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
self.assertEquals(results[0]['_source'].name, 'example')
def test_expected_match_results(self):
names = [
'Instrument for site1',
'Instrument for simulator',
'CTD1',
'SBE37',
'SSN-719',
'Submerssible Expendable Bathyothermograph',
'VELPT',
'VELO',
'Safire2 169'
]
for name in names:
res_id, _ = self.rr.create(InstrumentDevice(name=name))
self.addCleanup(self.rr.delete, res_id)
search_string = 'search "name" match "expendable" from "devices"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertEquals(len(results),1)
self.assertEquals(results[0]['_source'].name, 'Submerssible Expendable Bathyothermograph')
search_string = 'search "name" match "instrument for" from "devices"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertEquals(len(results),2)
self.assertTrue('Instrument for' in results[0]['_source'].name)
self.assertTrue('Instrument for' in results[1]['_source'].name)
search_string = 'search "name" match "velo for" from "devices"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertEquals(len(results),1)
self.assertEquals(results[0]['_source'].name, 'VELO')
def test_ownership_searching(self):
# Create two data products so that there is competition to the search, one is parsed
# (with conductivity as a parameter) and the other is raw
dp = DataProduct(name='example dataproduct')
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id)
dp_id = self.data_product_management.create_data_product(dp, stream_definition_id=stream_def_id, exchange_point='xp1')
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict')
stream_def_id = self.pubsub_management.create_stream_definition('ctd raw', parameter_dictionary_id=pdict_id)
dp = DataProduct(name='WRONG')
self.data_product_management.create_data_product(dp, stream_definition_id=stream_def_id, exchange_point='xp1')
parameter_search = 'search "name" is "conductivity" from "resources_index"'
results = self.poll(9, self.discovery.parse, parameter_search)
param_id = results[0]['_id']
data_product_search = 'search "name" is "*" from "data_products_index" and has "%s"' % param_id
results = self.poll(9, self.discovery.parse, data_product_search,id_only=True)
self.assertIn(dp_id, results)
#self.assertEquals(results[0], dp_id)
def test_associative_searching(self):
dp_id,_ = self.rr.create(DataProduct('test_foo'))
ds_id,_ = self.rr.create(Dataset('test_bar', registered=True))
self.rr.create_association(subject=dp_id, object=ds_id, predicate='hasDataset')
search_string = "search 'type_' is 'Dataset' from 'resources_index' and belongs to '%s'" % dp_id
results = self.poll(5, self.discovery.parse, search_string, id_only=True)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(ds_id in results)
def test_iterative_associative_searching(self):
#--------------------------------------------------------------------------------
# Tests the ability to limit the iterations
#--------------------------------------------------------------------------------
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
search_string = "belongs to '%s' depth 1" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
correct = [transform_id]
self.assertTrue(results == correct, '%s' % results)
search_string = "belongs to '%s' depth 2" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)