def setUp(self):
super(DiscoveryUnitTest, self).setUp()
mock_clients = self._create_service_mock('discovery')
self.discovery = DiscoveryService()
self.discovery.on_start()
self.discovery.clients = mock_clients
self.ds_mock = Mock()
self.discovery.ds_discovery._get_datastore = Mock(return_value=self.ds_mock)
container_mock = Mock()
self.discovery.ds_discovery.container = container_mock
container_mock.resource_registry = Mock()
container_mock.resource_registry.get_superuser_actors = Mock(return_value={})
self.rr_create = mock_clients.resource_registry.create
self.rr_read = mock_clients.resource_registry.read
self.rr_update = mock_clients.resource_registry.update
self.rr_delete = mock_clients.resource_registry.delete
self.rr_find_assoc = mock_clients.resource_registry.find_associations
self.rr_find_res = mock_clients.resource_registry.find_resources
self.rr_find_obj = mock_clients.resource_registry.find_objects
self.rr_find_assocs_mult = mock_clients.resource_registry.find_objects_mult
self.rr_create_assoc = mock_clients.resource_registry.create_association
self.rr_delete_assoc = mock_clients.resource_registry.delete_association
self.cms_create = mock_clients.catalog_management.create_catalog
self.cms_list_indexes = mock_clients.catalog_management.list_indexes
def setUp(self):
self._start_container()
self.discovery = DiscoveryService()
self.discovery.container = self.container
self.discovery.on_start()
self.rr = self.container.resource_registry
def setUp(self):
super(DiscoveryUnitTest,self).setUp()
mock_clients = self._create_service_mock('discovery')
self.discovery = DiscoveryService()
self.discovery.clients = mock_clients
self.discovery.use_es = True
self.rr_create = mock_clients.resource_registry.create
self.rr_read = mock_clients.resource_registry.read
self.rr_update = mock_clients.resource_registry.update
self.rr_delete = mock_clients.resource_registry.delete
self.rr_find_assoc = mock_clients.resource_registry.find_associations
self.rr_find_res = mock_clients.resource_registry.find_resources
self.rr_find_obj = mock_clients.resource_registry.find_objects
self.rr_create_assoc = mock_clients.resource_registry.create_association
self.cms_create = mock_clients.catalog_management.create_catalog
self.cms_list_indexes = mock_clients.catalog_management.list_indexes
def setUp(self):
super(DiscoveryUnitTest,self).setUp()
mock_clients = self._create_service_mock('discovery')
self.discovery = DiscoveryService()
self.discovery.clients = mock_clients
self.discovery.use_es = True
self.rr_create = mock_clients.resource_registry.create
self.rr_read = mock_clients.resource_registry.read
self.rr_update = mock_clients.resource_registry.update
self.rr_delete = mock_clients.resource_registry.delete
self.rr_find_assoc = mock_clients.resource_registry.find_associations
self.rr_find_res = mock_clients.resource_registry.find_resources
self.rr_find_obj = mock_clients.resource_registry.find_objects
self.rr_find_assocs_mult = mock_clients.resource_registry.find_objects_mult
self.rr_create_assoc = mock_clients.resource_registry.create_association
self.rr_delete_assoc = mock_clients.resource_registry.delete_association
self.cms_create = mock_clients.catalog_management.create_catalog
self.cms_list_indexes = mock_clients.catalog_management.list_indexes
class DiscoveryUnitTest(PyonTestCase):
def setUp(self):
super(DiscoveryUnitTest,self).setUp()
mock_clients = self._create_service_mock('discovery')
self.discovery = DiscoveryService()
self.discovery.clients = mock_clients
self.discovery.use_es = True
self.rr_create = mock_clients.resource_registry.create
self.rr_read = mock_clients.resource_registry.read
self.rr_update = mock_clients.resource_registry.update
self.rr_delete = mock_clients.resource_registry.delete
self.rr_find_assoc = mock_clients.resource_registry.find_associations
self.rr_find_res = mock_clients.resource_registry.find_resources
self.rr_find_obj = mock_clients.resource_registry.find_objects
self.rr_find_assocs_mult = mock_clients.resource_registry.find_objects_mult
self.rr_create_assoc = mock_clients.resource_registry.create_association
self.rr_delete_assoc = mock_clients.resource_registry.delete_association
self.cms_create = mock_clients.catalog_management.create_catalog
self.cms_list_indexes = mock_clients.catalog_management.list_indexes
def test_create_view(self):
# Mocks
self.rr_find_res.return_value = ([],[])
self.rr_create.return_value = ('res_id', 'rev_id')
self.cms_create.return_value = 'catalog_id'
retval = self.discovery.create_view('mock_view',fields=['name'])
self.assertTrue(retval=='res_id', 'Improper resource creation')
def test_create_exists(self):
self.rr_find_res.return_value = ([1], [1])
with self.assertRaises(BadRequest):
self.discovery.create_view('doesnt matter', fields=['name'])
def test_create_no_fields(self):
self.rr_find_res.return_value = ([],[])
self.rr_create.return_value = ('res_id', 'rev_id')
self.cms_create.return_value = 'catalog_id'
with self.assertRaises(BadRequest):
self.discovery.create_view('mock_view')
def test_create_order(self):
self.rr_find_res.return_value = ([],[])
self.rr_create.return_value = ('res_id', 'rev_id')
self.cms_create.return_value = 'catalog_id'
with self.assertRaises(BadRequest):
self.discovery.create_view('movk_view',fields=['name'],order=['age'])
def test_read_view(self):
# Mocks
self.rr_read.return_value = 'fake_resource'
retval = self.discovery.read_view('mock_view_id')
self.assertTrue(retval == 'fake_resource')
def test_update_view(self):
retval = self.discovery.update_view({})
self.assertTrue(retval)
self.rr_update.assert_called_once_with({})
def test_delete_view(self):
catalog_assoc = DotDict(_id=0)
self.rr_find_assocs_mult.return_value = ([],[catalog_assoc])
retval = self.discovery.delete_view('view_id')
self.assertTrue(retval)
self.rr_delete.assert_called_once_with('view_id')
self.assertTrue(self.rr_delete_assoc.call_count == 1)
def test_list_catalogs(self):
self.rr_find_obj.return_value = (['test'],[])
retval = self.discovery.list_catalogs('view_id')
self.assertTrue(retval[0] == 'test')
@patch('ion.services.dm.presentation.discovery_service.ep.ElasticSearch')
def test_query_index(self, es_mock):
mock_index = ElasticSearchIndex(content_type=IndexManagementService.ELASTICSEARCH_INDEX)
self.rr_read.return_value = mock_index
self.discovery.elasticsearch_host = 'fakehost'
self.discovery.elasticsearch_port = 'fakeport'
es_mock().search_index_advanced.return_value = {'hits':{'hits':[{'_id':'success'}]}}
retval = self.discovery.query_term('mock_index', 'field', 'value', order={'name':'asc'}, limit=20, offset=20)
self.assertTrue(retval[0]['_id']=='success', '%s' % retval)
def test_query(self):
self.discovery.request = lambda x : x
retval = self.discovery.query('test')
self.assertTrue(retval == 'test')
def test_query_couch(self):
pass
@skip('Needs to be adjusted for new traversal')
def test_traverse(self):
edge_list = ['B','C','D']
callers = []
def pop_edge(*args, **kwargs):
callers.append(args[0])
if len(edge_list):
val = edge_list.pop(0)
return [val], 'blah'
return [], 'blah'
self.rr_find_obj.side_effect = pop_edge
retval = self.discovery.traverse('A')
retval.sort()
self.assertTrue(retval == ['B','C','D'], '%s' % retval)
def test_intersect(self):
test_vals = [0,1,2,3]
other = [2,1]
retval = self.discovery.intersect(test_vals,other)
retval.sort()
self.assertTrue(retval == [1,2])
def test_union(self):
test_vals = [0,1,2,3]
other = [4,5]
retval = self.discovery.union(test_vals,other)
retval.sort()
self.assertTrue(retval == [0,1,2,3,4,5], '%s' % retval)
@patch('ion.services.dm.presentation.discovery_service.QueryLanguage')
def test_parse(self, mock_parser):
mock_parser().parse.return_value = 'arg'
self.discovery.request = Mock()
self.discovery.request.return_value = 'correct_value'
retval = self.discovery.parse('blah blah')
self.discovery.request.assert_called_once_with('arg')
self.assertTrue(retval=='correct_value', '%s' % retval)
def test_query_request_term_search(self):
query = DotDict()
query.index = 'index_id'
query.field = 'field'
query.value = 'value'
self.discovery.query_term = Mock()
self.discovery.query_term.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test', '%s' % retval)
def test_query_request_association(self):
query = DotDict()
query.association = 'resource_id'
self.discovery.query_association = Mock()
self.discovery.query_association.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test')
def test_query_request_range(self):
query = DotDict()
query['range'] = {'from':0, 'to':90}
query.index = 'index_id'
query.field = 'field'
self.discovery.query_range = Mock()
self.discovery.query_range.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test')
def test_query_request_collection(self):
query = DotDict()
query.collection = 'test'
self.discovery.query_collection = Mock()
self.discovery.query_collection.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test')
def test_bad_query(self):
query = DotDict()
query.unknown = 'yup'
with self.assertRaises(BadRequest):
self.discovery.query_request(query)
@patch('ion.services.dm.presentation.discovery_service.ep.ElasticSearch')
def test_query_range(self, mock_es):
mock_index = ElasticSearchIndex(name='index', index_name='index')
self.discovery.elasticsearch_host = 'fakehost'
self.discovery.elasticsearch_port = 'fakeport'
self.rr_read.return_value = mock_index
hits = [{'_id':'a'},{'_id':'b'}]
mock_es().search_index_advanced.return_value = {'hits':{'hits':hits}}
retval = self.discovery.query_range('index_id','field',0,100,id_only=False)
mock_es().search_index_advanced.assert_called_once_with('index',ep.ElasticQuery.range(field='field', from_value=0, to_value=100))
retval.sort()
self.assertTrue(retval==hits, '%s' % retval)
retval = self.discovery.query_range('index_id','field',0,100,id_only=True)
retval.sort()
self.assertTrue(retval==['a','b'])
@patch('ion.services.dm.presentation.discovery_service.ep.ElasticSearch')
def test_query_time(self, mock_es):
mock_index = ElasticSearchIndex(name='index', index_name='index')
self.discovery.elasticsearch_host = 'fakehost'
self.discovery.elasticsearch_port = 'fakeport'
self.rr_read.return_value = mock_index
hits = [{'_id':'a'},{'_id':'b'}]
mock_es().search_index_advanced.return_value = {'hits':{'hits':hits}}
date1 = '2012-01-01'
ts1 = time.mktime( dateutil.parser.parse(date1).timetuple()) * 1000
date2 = '2012-02-01'
ts2 = time.mktime( dateutil.parser.parse(date2).timetuple()) * 1000
retval = self.discovery.query_time('index_id','field',date1,date2,id_only=False)
mock_es().search_index_advanced.assert_called_once_with('index',ep.ElasticQuery.range(field='field', from_value=ts1, to_value=ts2))
retval.sort()
self.assertTrue(retval==hits, '%s' % retval)
retval = self.discovery.query_time('index_id','field',date1,date2,id_only=True)
retval.sort()
self.assertTrue(retval==['a','b'])
@skip('Needs to be adjusted for changes in association traversal')
def test_query_association(self):
self.discovery.traverse = Mock()
self.discovery.traverse.return_value = ['a','b','c']
retval = self.discovery.query_association('blah',id_only=True)
retval.sort()
self.assertTrue(retval == ['a','b','c'])
self.rr_read.return_value = 'test'
retval = self.discovery.query_association('blah',id_only=False)
self.assertTrue(retval == (['test']*3))
def test_es_map_query(self):
pass
def test_tier1_request(self):
self.discovery.query_request = Mock()
self.discovery.query_request.return_value = 'test'
query = {'and':[], 'or':[], 'query':{}}
retval = self.discovery.request(query)
self.assertTrue(retval=='test')
self.discovery.query_request.assert_called_once_with({})
def test_tier2_request(self):
result_list = [[0,1,2],[1,2],[0,1,2],[1,2,3,4]]
def query_request(*args, **kwargs):
if len(result_list):
return result_list.pop(0)
return None
self.discovery.query_request = Mock()
self.discovery.query_request.side_effect = query_request
#========================
# Intersection
#========================
request = {'and':[{}], 'or':[], 'query':{}}
retval = self.discovery.request(request)
retval.sort()
self.assertTrue(retval == [1,2])
#========================
# Union
#========================
request = {'and':[], 'or':[{}], 'query':{}}
retval = self.discovery.request(request)
retval.sort()
self.assertTrue(retval == [0,1,2,3,4])
def test_bad_requests(self):
#================================
# Battery of broken requests
#================================
bad_requests = [
{},
{'and':[]},
{'query':{}},
{'or':[]},
{'and':{},'or':[],'query':[]},
]
for req in bad_requests:
with self.assertRaises(BadRequest):
self.discovery.request(req)
@patch('ion.services.dm.presentation.discovery_service.ep.ElasticSearch')
def test_view_request(self, mock_es):
self.call_count = 0
def cb(*args, **kwargs):
self.call_count +=1
return [1]
self.discovery.elasticsearch_host = ''
self.discovery.elasticsearch_port = ''
v = View()
setattr(v, '_id', 'a')
self.discovery.list_catalogs = Mock()
self.discovery.list_catalogs.return_value = [1,2]
retval = self.discovery._multi(cb,v)
self.assertTrue(retval == [1,1])
c = Catalog()
setattr(c, '_id', 'c')
self.cms_list_indexes.return_value = [1,2]
retval = self.discovery._multi(cb,c)
self.assertTrue(retval == [1,1])
retval = self.discovery._multi(cb,v, limit=1)
self.assertTrue(retval == [1])
retval = self.discovery._multi(cb,c, limit=1)
self.assertTrue(retval == [1])
self.discovery._multi = Mock()
self.rr_read.return_value = v
self.discovery._multi.return_value = 'test'
retval = self.discovery.query_term('blah', 'field', 'value')
self.assertTrue(retval == 'test')
@patch('ion.services.dm.presentation.discovery_service.ep.ElasticSearch')
def test_query_geo_distance(self, mock_es):
self.rr_read.return_value = ElasticSearchIndex(name='test')
self.discovery.elasticsearch_host = ''
self.discovery.elasticsearch_port = ''
self.discovery._multi = Mock()
self.discovery._multi.return_value = None
response = {'ok':True, 'status':200, 'hits':{'hits':['hi']}}
mock_es().search_index_advanced.return_value = response
retval = self.discovery.query_geo_distance('abc13', 'blah', [0,0], 20)
self.assertTrue(retval == ['hi'])
@patch('ion.services.dm.presentation.discovery_service.ep.ElasticSearch')
def test_query_geo_bbox(self, mock_es):
self.rr_read.return_value = ElasticSearchIndex(name='test')
self.discovery.elasticsearch_host = ''
self.discovery.elasticsearch_port = ''
self.discovery._multi = Mock()
self.discovery._multi.return_value = None
response = {'ok':True, 'status':200, 'hits':{'hits':['hi']}}
mock_es().search_index_advanced.return_value = response
retval = self.discovery.query_geo_bbox('abc123', 'blah', [0,10], [10,0])
self.assertTrue(retval == ['hi'])
class DiscoveryUnitTest(PyonTestCase):
def setUp(self):
super(DiscoveryUnitTest,self).setUp()
mock_clients = self._create_service_mock('discovery')
self.discovery = DiscoveryService()
self.discovery.clients = mock_clients
self.discovery.use_es = True
self.rr_create = mock_clients.resource_registry.create
self.rr_read = mock_clients.resource_registry.read
self.rr_update = mock_clients.resource_registry.update
self.rr_delete = mock_clients.resource_registry.delete
self.rr_find_assoc = mock_clients.resource_registry.find_associations
self.rr_find_res = mock_clients.resource_registry.find_resources
self.rr_find_obj = mock_clients.resource_registry.find_objects
self.rr_create_assoc = mock_clients.resource_registry.create_association
self.cms_create = mock_clients.catalog_management.create_catalog
self.cms_list_indexes = mock_clients.catalog_management.list_indexes
def test_create_view(self):
# Mocks
self.rr_find_res.return_value = ([],[])
self.rr_create.return_value = ('res_id', 'rev_id')
self.cms_create.return_value = 'catalog_id'
retval = self.discovery.create_view('mock_view',fields=['name'])
self.assertTrue(retval=='res_id', 'Improper resource creation')
def test_create_exists(self):
self.rr_find_res.return_value = ([1], [1])
with self.assertRaises(BadRequest):
self.discovery.create_view('doesnt matter', fields=['name'])
def test_create_no_fields(self):
self.rr_find_res.return_value = ([],[])
self.rr_create.return_value = ('res_id', 'rev_id')
self.cms_create.return_value = 'catalog_id'
with self.assertRaises(BadRequest):
self.discovery.create_view('mock_view')
def test_create_order(self):
self.rr_find_res.return_value = ([],[])
self.rr_create.return_value = ('res_id', 'rev_id')
self.cms_create.return_value = 'catalog_id'
with self.assertRaises(BadRequest):
self.discovery.create_view('movk_view',fields=['name'],order=['age'])
def test_read_view(self):
# Mocks
self.rr_read.return_value = 'fake_resource'
retval = self.discovery.read_view('mock_view_id')
self.assertTrue(retval == 'fake_resource')
def test_update_view(self):
retval = self.discovery.update_view({})
self.assertTrue(retval)
self.rr_update.assert_called_once_with({})
def test_delete_view(self):
retval = self.discovery.delete_view('view_id')
self.assertTrue(retval)
self.rr_delete.assert_called_once_with('view_id')
def test_list_catalogs(self):
self.rr_find_obj.return_value = (['test'],[])
retval = self.discovery.list_catalogs('view_id')
self.assertTrue(retval[0] == 'test')
@patch('ion.services.dm.presentation.discovery_service.ep.ElasticSearch')
def test_query_index(self, es_mock):
mock_index = ElasticSearchIndex(content_type=IndexManagementService.ELASTICSEARCH_INDEX)
self.rr_read.return_value = mock_index
self.discovery.elasticsearch_host = 'fakehost'
self.discovery.elasticsearch_port = 'fakeport'
es_mock().search_index_advanced.return_value = {'hits':{'hits':[{'_id':'success'}]}}
retval = self.discovery.query_term('mock_index', 'field', 'value', order={'name':'asc'}, limit=20, offset=20)
self.assertTrue(retval[0]['_id']=='success', '%s' % retval)
def test_query(self):
self.discovery.request = lambda x : x
retval = self.discovery.query('test')
self.assertTrue(retval == 'test')
def test_query_couch(self):
pass
@skip('Needs to be adjusted for new traversal')
def test_traverse(self):
edge_list = ['B','C','D']
callers = []
def pop_edge(*args, **kwargs):
callers.append(args[0])
if len(edge_list):
val = edge_list.pop(0)
return [val], 'blah'
return [], 'blah'
self.rr_find_obj.side_effect = pop_edge
retval = self.discovery.traverse('A')
retval.sort()
self.assertTrue(retval == ['B','C','D'], '%s' % retval)
def test_intersect(self):
test_vals = [0,1,2,3]
other = [2,1]
retval = self.discovery.intersect(test_vals,other)
retval.sort()
self.assertTrue(retval == [1,2])
def test_union(self):
test_vals = [0,1,2,3]
other = [4,5]
retval = self.discovery.union(test_vals,other)
retval.sort()
self.assertTrue(retval == [0,1,2,3,4,5], '%s' % retval)
@patch('ion.services.dm.presentation.discovery_service.QueryLanguage')
def test_parse(self, mock_parser):
mock_parser().parse.return_value = 'arg'
self.discovery.request = Mock()
self.discovery.request.return_value = 'correct_value'
retval = self.discovery.parse('blah blah')
self.discovery.request.assert_called_once_with('arg')
self.assertTrue(retval=='correct_value', '%s' % retval)
def test_query_request_term_search(self):
query = DotDict()
query.index = 'index_id'
query.field = 'field'
query.value = 'value'
self.discovery.query_term = Mock()
self.discovery.query_term.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test', '%s' % retval)
def test_query_request_association(self):
query = DotDict()
query.association = 'resource_id'
self.discovery.query_association = Mock()
self.discovery.query_association.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test')
def test_query_request_range(self):
query = DotDict()
query['range'] = {'from':0, 'to':90}
query.index = 'index_id'
query.field = 'field'
self.discovery.query_range = Mock()
self.discovery.query_range.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test')
def test_query_request_collection(self):
query = DotDict()
query.collection = 'test'
self.discovery.query_collection = Mock()
self.discovery.query_collection.return_value = 'test'
retval = self.discovery.query_request(query)
self.assertTrue(retval == 'test')
def test_bad_query(self):
query = DotDict()
query.unknown = 'yup'
with self.assertRaises(BadRequest):
self.discovery.query_request(query)
@patch('ion.services.dm.presentation.discovery_service.ep.ElasticSearch')
def test_query_range(self, mock_es):
mock_index = ElasticSearchIndex(name='index', index_name='index')
self.discovery.elasticsearch_host = 'fakehost'
self.discovery.elasticsearch_port = 'fakeport'
self.rr_read.return_value = mock_index
hits = [{'_id':'a'},{'_id':'b'}]
mock_es().search_index_advanced.return_value = {'hits':{'hits':hits}}
retval = self.discovery.query_range('index_id','field',0,100,id_only=False)
mock_es().search_index_advanced.assert_called_once_with('index',ep.ElasticQuery().range(field='field', from_value=0, to_value=100))
retval.sort()
self.assertTrue(retval==hits, '%s' % retval)
retval = self.discovery.query_range('index_id','field',0,100,id_only=True)
retval.sort()
self.assertTrue(retval==['a','b'])
@skip('Needs to be adjusted for changes in association traversal')
def test_query_association(self):
self.discovery.traverse = Mock()
self.discovery.traverse.return_value = ['a','b','c']
retval = self.discovery.query_association('blah',id_only=True)
retval.sort()
self.assertTrue(retval == ['a','b','c'])
self.rr_read.return_value = 'test'
retval = self.discovery.query_association('blah',id_only=False)
self.assertTrue(retval == (['test']*3))
def test_es_map_query(self):
pass
def test_tier1_request(self):
self.discovery.query_request = Mock()
self.discovery.query_request.return_value = 'test'
query = {'and':[], 'or':[], 'query':{}}
retval = self.discovery.request(query)
self.assertTrue(retval=='test')
self.discovery.query_request.assert_called_once_with({})
def test_tier2_request(self):
result_list = [[0,1,2],[1,2],[0,1,2],[1,2,3,4]]
def query_request(*args, **kwargs):
if len(result_list):
return result_list.pop(0)
return None
self.discovery.query_request = Mock()
self.discovery.query_request.side_effect = query_request
#========================
# Intersection
#========================
request = {'and':[{}], 'or':[], 'query':{}}
retval = self.discovery.request(request)
retval.sort()
self.assertTrue(retval == [1,2])
#========================
# Union
#========================
request = {'and':[], 'or':[{}], 'query':{}}
retval = self.discovery.request(request)
retval.sort()
self.assertTrue(retval == [0,1,2,3,4])
def test_bad_requests(self):
#================================
# Battery of broken requests
#================================
bad_requests = [
{},
{'and':[]},
{'query':{}},
{'or':[]},
{'and':{},'or':[],'query':[]},
]
for req in bad_requests:
with self.assertRaises(BadRequest):
self.discovery.request(req)
@patch('ion.services.dm.presentation.discovery_service.ep.ElasticSearch')
def test_view_request(self, mock_es):
self.call_count = 0
def cb(*args, **kwargs):
self.call_count +=1
return [1]
self.discovery.elasticsearch_host = ''
self.discovery.elasticsearch_port = ''
v = View()
setattr(v, '_id', 'a')
self.discovery.list_catalogs = Mock()
self.discovery.list_catalogs.return_value = [1,2]
retval = self.discovery._multi(cb,v)
self.assertTrue(retval == [1,1])
c = Catalog()
setattr(c, '_id', 'c')
self.cms_list_indexes.return_value = [1,2]
retval = self.discovery._multi(cb,c)
self.assertTrue(retval == [1,1])
retval = self.discovery._multi(cb,v, limit=1)
self.assertTrue(retval == [1])
retval = self.discovery._multi(cb,c, limit=1)
self.assertTrue(retval == [1])
self.discovery._multi = Mock()
self.rr_read.return_value = v
self.discovery._multi.return_value = 'test'
retval = self.discovery.query_term('blah', 'field', 'value')
self.assertTrue(retval == 'test')
@patch('ion.services.dm.presentation.discovery_service.ep.ElasticSearch')
def test_query_geo_distance(self, mock_es):
self.rr_read.return_value = ElasticSearchIndex(name='test')
self.discovery.elasticsearch_host = ''
self.discovery.elasticsearch_port = ''
self.discovery._multi = Mock()
self.discovery._multi.return_value = None
response = {'ok':True, 'status':200, 'hits':{'hits':['hi']}}
mock_es().search_index_advanced.return_value = response
retval = self.discovery.query_geo_distance('abc13', 'blah', [0,0], 20)
self.assertTrue(retval == ['hi'])
@patch('ion.services.dm.presentation.discovery_service.ep.ElasticSearch')
def test_query_geo_bbox(self, mock_es):
self.rr_read.return_value = ElasticSearchIndex(name='test')
self.discovery.elasticsearch_host = ''
self.discovery.elasticsearch_port = ''
self.discovery._multi = Mock()
self.discovery._multi.return_value = None
response = {'ok':True, 'status':200, 'hits':{'hits':['hi']}}
mock_es().search_index_advanced.return_value = response
retval = self.discovery.query_geo_bbox('abc123', 'blah', [0,10], [10,0])
self.assertTrue(retval == ['hi'])
class DiscoveryUnitTest(IonUnitTestCase):
def setUp(self):
super(DiscoveryUnitTest, self).setUp()
mock_clients = self._create_service_mock('discovery')
self.discovery = DiscoveryService()
self.discovery.on_start()
self.discovery.clients = mock_clients
self.ds_mock = Mock()
self.discovery.ds_discovery._get_datastore = Mock(return_value=self.ds_mock)
container_mock = Mock()
self.discovery.ds_discovery.container = container_mock
container_mock.resource_registry = Mock()
container_mock.resource_registry.get_superuser_actors = Mock(return_value={})
self.rr_create = mock_clients.resource_registry.create
self.rr_read = mock_clients.resource_registry.read
self.rr_update = mock_clients.resource_registry.update
self.rr_delete = mock_clients.resource_registry.delete
self.rr_find_assoc = mock_clients.resource_registry.find_associations
self.rr_find_res = mock_clients.resource_registry.find_resources
self.rr_find_obj = mock_clients.resource_registry.find_objects
self.rr_find_assocs_mult = mock_clients.resource_registry.find_objects_mult
self.rr_create_assoc = mock_clients.resource_registry.create_association
self.rr_delete_assoc = mock_clients.resource_registry.delete_association
self.cms_create = mock_clients.catalog_management.create_catalog
self.cms_list_indexes = mock_clients.catalog_management.list_indexes
def test_create_view(self):
# Mocks
self.rr_find_res.return_value = ([],[])
self.rr_create.return_value = ('res_id', 'rev_id')
self.cms_create.return_value = 'catalog_id'
retval = self.discovery.create_view('mock_view',fields=['name'])
self.assertTrue(retval=='res_id', 'Improper resource creation')
def test_create_view_exists(self):
self.rr_find_res.return_value = ([1], [1])
with self.assertRaises(BadRequest):
self.discovery.create_view('doesnt matter', fields=['name'])
def test_create_view_no_fields(self):
self.rr_find_res.return_value = ([],[])
self.rr_create.return_value = ('res_id', 'rev_id')
self.cms_create.return_value = 'catalog_id'
with self.assertRaises(BadRequest):
self.discovery.create_view('mock_view')
def test_create_view_order(self):
self.rr_find_res.return_value = ([],[])
self.rr_create.return_value = ('res_id', 'rev_id')
self.cms_create.return_value = 'catalog_id'
with self.assertRaises(BadRequest):
self.discovery.create_view('movk_view',fields=['name'],order=['age'])
def test_read_view(self):
# Mocks
self.rr_read.return_value = 'fake_resource'
retval = self.discovery.read_view('mock_view_id')
self.assertTrue(retval == 'fake_resource')
def test_update_view(self):
retval = self.discovery.update_view({})
self.assertTrue(retval)
self.rr_update.assert_called_once_with({})
def test_delete_view(self):
catalog_assoc = DotDict(_id=0)
self.rr_find_assocs_mult.return_value = ([],[catalog_assoc])
retval = self.discovery.delete_view('view_id')
self.assertTrue(retval)
self.rr_delete.assert_called_once_with('view_id')
self.assertTrue(self.rr_delete_assoc.call_count == 1)
def test_bad_query(self):
query = DotDict()
query.unknown = 'yup'
with self.assertRaises(BadRequest):
self.discovery.query(query)
def test_bad_requests(self):
#================================
# Battery of broken requests
#================================
self.discovery.ds_discovery = Mock()
bad_requests = [
{},
{'field':"f"},
{'and':[]},
{'or':[]},
]
for req in bad_requests:
with self.assertRaises(BadRequest):
self.discovery.request(req)
@patch('ion.services.dm.presentation.discovery_service.QueryLanguage')
def test_parse_mock(self, mock_parser):
mock_parser().parse.return_value = 'arg'
self.discovery.request = Mock()
self.discovery.request.return_value = 'correct_value'
retval = self.discovery.parse('blah blah', id_only=sentinel.id_only)
self.discovery.request.assert_called_once_with('arg', search_args=None, id_only=sentinel.id_only)
self.assertTrue(retval=='correct_value', '%s' % retval)
def test_parse(self):
self.ds_mock.find_by_query = Mock(return_value=["FOO"])
search_string = "search 'serial_number' is 'abc' from 'resources_index'"
retval = self.discovery.parse(search_string)
self.discovery.ds_discovery._get_datastore.assert_called_once_with("resources")
self.assertEquals(retval, ["FOO"])
def test_tier1_request(self):
self.ds_mock.find_by_query = Mock(return_value=["FOO"])
query = {'query':{'field': 'name', 'value': 'foo'}}
retval = self.discovery.request(query)
self.discovery.ds_discovery._get_datastore.assert_called_once_with("resources")
self.assertEquals(retval, ["FOO"])
def test_tier2_request(self):
self.ds_mock.find_by_query = Mock(return_value=["FOO"])
query = {'query':{'field': 'name', 'value': 'foo'}, 'and':[{'field': 'lcstate', 'value': 'foo2'}]}
retval = self.discovery.request(query)
self.discovery.ds_discovery._get_datastore.assert_called_once_with("resources")
self.assertEquals(retval, ["FOO"])
query = {'query':{'field': 'name', 'value': 'foo'}, 'or':[{'field': 'lcstate', 'value': 'foo2'}]}
retval = self.discovery.request(query)
self.assertEquals(retval, ["FOO"])
class DiscoveryQueryTest(IonIntegrationTestCase):
"""Tests discovery in a somewhat integration environment. Only a container and a DiscoveryService instance
but no r2deploy and process"""
def setUp(self):
self._start_container()
self.discovery = DiscoveryService()
self.discovery.container = self.container
self.discovery.on_start()
self.rr = self.container.resource_registry
def _geopt(self, x1, y1):
return GeospatialIndex(lat=float(x1), lon=float(y1))
def _geobb(self, x1, y1, x2=None, y2=None, z1=0.0, z2=None):
if x2 is None: x2 = x1
if y2 is None: y2 = y1
if z2 is None: z2 = z1
return GeospatialBounds(geospatial_latitude_limit_north=float(y2),
geospatial_latitude_limit_south=float(y1),
geospatial_longitude_limit_west=float(x1),
geospatial_longitude_limit_east=float(x2),
geospatial_vertical_min=float(z1),
geospatial_vertical_max=float(z2))
def _temprng(self, t1="", t2=None):
if t2 is None: t2 = t1
return TemporalBounds(start_datetime=str(t1), end_datetime=str(t2))
def _geodp(self, x1, y1, x2=None, y2=None, z1=0.0, z2=None, t1="", t2=None):
if x2 is None: x2 = x1
if y2 is None: y2 = y1
if z2 is None: z2 = z1
if t2 is None: t2 = t1
bounds = self._geobb(x1, y1, x2, y2, z1, z2)
attrs = dict(geospatial_point_center=GeoUtils.calc_geospatial_point_center(bounds),
geospatial_bounds=bounds,
nominal_datetime=self._temprng(t1, t2))
return attrs
def test_basic_searching(self):
t0 = 1363046400
hour = 60*24
day = 60*60*24
resources = [
("ID1", InstrumentDevice(name='sonobuoy1', firmware_version='A1')),
("ID2", InstrumentDevice(name='sonobuoy2', firmware_version='A2')),
("ID3", InstrumentDevice(name='sonobuoy3', firmware_version='A3')),
("DP1", DataProduct(name='testData1', **self._geodp(5, 5, 15, 15, 0, 100, t0, t0+day))),
("DP2", DataProduct(name='testData2', **self._geodp(25, 5, 35, 15, 0, 100, t0+hour+day, t0+2*day))),
("DP3", DataProduct(name='testData3', **self._geodp(30, 10, 40, 20, 50, 200, t0+100*day, t0+110*day))),
("DP4", DataProduct(name='testData4', **self._geodp(30, 5, 32, 10, 5, 20, t0+100*day, t0+110*day))),
]
# create a large range of resources to test skip(offset)
for i in range(200):
resources.append(("INS%03d" % i, InstrumentDevice(name='range%03d' % i)))
res_by_alias = {}
for (alias, resource) in resources:
rid,_ = self.rr.create(resource)
res_by_alias[alias] = rid
# ----------------------------------------------------
# Resource attribute search
# Resource attribute equals
search_string = "search 'firmware_version' is 'A2' from 'resources_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 1)
self.assertIsInstance(result[0], InstrumentDevice)
self.assertTrue(result[0].name == 'sonobuoy2')
self.assertTrue(result[0].firmware_version == 'A2')
query_str = "{'and': [], 'or': [], 'query': {'field': 'firmware_version', 'index': 'resources_index', 'value': 'A2'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result), len(result1))
self.assertEquals(result, result1)
# Resource attribute match
search_string = "search 'firmware_version' is 'A*' from 'resources_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 3)
query_str = "{'and': [], 'or': [], 'query': {'field': 'firmware_version', 'index': 'resources_index', 'value': 'A*'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result), len(result1))
self.assertEquals(result, result1)
# Resource attribute match with limit
search_string = "search 'firmware_version' is 'A*' from 'resources_index' limit 2"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 2)
query_str = "{'and': [], 'limit': 2, 'or': [], 'query': {'field': 'firmware_version', 'index': 'resources_index', 'value': 'A*'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result), len(result1))
self.assertEquals(result, result1)
# Resource attribute match with limit and skip (offset)
# TODO these tests are unlikely to always work until order_by is implemented (1/200 chance fails)
# -- limit 1 without skip (using QueryLanguage)
search_string = "search 'name' is 'range*' from 'resources_index' limit 1"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 1)
# -- limit 1 with skip (using QueryLanguage)
search_string = "search 'name' is 'range*' from 'resources_index' SKIP 100 limit 1"
result1 = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result1), 1)
# check same length and not equal (one uses SKIP 100, other doesn't)
self.assertEquals(len(result), len(result1))
self.assertNotEquals(result, result1)
# -- limit 1 without skip (using Discovery Intermediate Format)
query_str = "{'and': [], 'limit': 1, 'or': [], 'query': {'field': 'name', 'index': 'resources_index', 'value': 'range*'}}"
query_obj = eval(query_str)
result = self.discovery.query(query_obj, id_only=False)
# -- limit 1 with skip (using Discovery Intermediate Format)
query_str = "{'and': [], 'limit': 1, 'skip': 100, 'or': [], 'query': {'field': 'name', 'index': 'resources_index', 'value': 'range*'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False)
# check same length and not equal (one uses SKIP 100, other doesn't)
self.assertEquals(len(result), len(result1))
self.assertNotEquals(result, result1)
# Resource attribute match only count (results should return single value, a count of available results)
search_args_str = "{'count': True}"
search_args = eval(search_args_str)
search_string = "search 'firmware_version' is 'A*' from 'resources_index' limit 2"
result = self.discovery.parse(search_string, id_only=False, search_args=search_args)
self.assertEquals(len(result), 1)
query_str = "{'and': [], 'limit': 2, 'or': [], 'query': {'field': 'firmware_version', 'index': 'resources_index', 'value': 'A*'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False, search_args=search_args)
self.assertEquals(len(result), len(result1))
self.assertEquals(result, result1)
# Check data products
search_string = "search 'name' is 'testData*' from 'resources_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 4)
search_string = "search 'type_' is 'DataProduct' from 'resources_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 4)
# ----------------------------------------------------
# Geospatial search
# Geospatial search - query bbox fully overlaps
search_string = "search 'geospatial_point_center' geo box top-left lat 180 lon -180 bottom-right lat -180 lon 180 from 'resources_index'"
result = self.discovery.parse(search_string, id_only=True)
self.assertGreaterEqual(len(result), 4)
for dp in ["DP1", "DP2", "DP3", "DP4"]:
self.assertIn(res_by_alias[dp], result)
search_string = "search 'geospatial_point_center' geo box top-left lat 20 lon 0 bottom-right lat 0 lon 20 from 'resources_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 1)
# Note that in Discovery intermediate format top_left=x1,y2 and bottom_right=x2,y1 contrary to naming
query_str = "{'and': [], 'or': [], 'query': {'top_left': [0.0, 20.0], 'bottom_right': [20.0, 0.0], 'field': 'index_location', 'index': 'resources_index'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result), len(result1))
self.assertEquals(result, result1)
# Geospatial bbox operators - overlaps (this is the default and should be the same as above)
query_str = "{'and': [], 'or': [], 'query': {'top_left': [0.0, 20.0], 'bottom_right': [20.0, 0.0], 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'overlaps'}}"
query_obj = eval(query_str)
result2 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result1), len(result2))
self.assertEquals(result1, result2)
# Geospatial bbox operators - contains (the resource contains the query)
query_str = "{'and': [], 'or': [], 'query': {'top_left': [0.0, 20.0], 'bottom_right': [20.0, 0.0], 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'contains'}}"
query_obj = eval(query_str)
result3 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result3), 0)
query_str = "{'and': [], 'or': [], 'query': {'top_left': [8.0, 11.0], 'bottom_right': [12.0, 9.0], 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'contains'}}"
query_obj = eval(query_str)
result3 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result3), 1)
# Geospatial bbox operators - within (the resource with the query)
query_str = "{'and': [], 'or': [], 'query': {'top_left': [0.0, 20.0], 'bottom_right': [20.0, 0.0], 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'within'}}"
query_obj = eval(query_str)
result3 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result3), 1)
query_str = "{'and': [], 'or': [], 'query': {'top_left': [15.0, 5.0], 'bottom_right': [5.0, 15.0], 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'within'}}"
query_obj = eval(query_str)
result3 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result3), 1)
query_str = "{'and': [], 'or': [], 'query': {'top_left': [14.0, 5.0], 'bottom_right': [5.0, 15.0], 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'within'}}"
query_obj = eval(query_str)
result3 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result3), 0)
# Geospatial search - query bbox partial overlaps
search_string = "search 'geospatial_bounds' geo box top-left lat 11 lon 9 bottom-right lat 9 lon 11 from 'resources_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 1)
# Geospatial - WKT (a box 4,4 to 4,14 to 14,14 to 14,4, to 4,4 overlaps DP1 but is not contained by it or does not have it within)
query_str = "{'and': [], 'or': [], 'query': {'wkt': 'POLYGON((4 4,4 14,14 14,14 4,4 4))', 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'overlaps'}}"
query_obj = eval(query_str)
result = self.discovery.query(query_obj, id_only=False)
print result
self.assertEquals(len(result), 1)
query_str = "{'and': [], 'or': [], 'query': {'wkt': 'POLYGON((4 4,4 14,14 14,14 4,4 4))', 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'contains'}}"
query_obj = eval(query_str)
result = self.discovery.query(query_obj, id_only=False)
print result
self.assertEquals(len(result), 0)
query_str = "{'and': [], 'or': [], 'query': {'wkt': 'POLYGON((4 4,4 14,14 14,14 4,4 4))', 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'within'}}"
query_obj = eval(query_str)
result = self.discovery.query(query_obj, id_only=False)
print result
self.assertEquals(len(result), 0)
# -- with buffer (eg. point with radius CIRCLE)
query_str = "{'and': [], 'or': [], 'query': {'wkt': 'POINT(10.0 10.0)', 'buffer': 1.0, 'field': 'geospatial_point_center', 'index': 'resources_index', 'cmpop': 'within'}}"
query_obj = eval(query_str)
result = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result), 1)
query_str = "{'and': [], 'or': [], 'query': {'wkt': 'POINT(10.0 10.0)', 'buffer': 1.0, 'field': 'geospatial_point_center', 'index': 'resources_index', 'cmpop': 'contains'}}"
query_obj = eval(query_str)
result = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result), 0)
# ----------------------------------------------------
# Vertical search
search_string = "search 'geospatial_bounds' vertical from 0 to 500 from 'resources_index'"
result = self.discovery.parse(search_string, id_only=True)
self.assertGreaterEqual(len(result), 4)
for dp in ["DP1", "DP2", "DP3", "DP4"]:
self.assertIn(res_by_alias[dp], result)
query_str = "{'and': [], 'or': [], 'query': {'field': 'geospatial_bounds', 'index': 'resources_index', 'vertical_bounds': {'from': 0.0, 'to': 500.0}, 'cmpop': 'overlaps'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result1), 4)
query_str = "{'and': [], 'or': [], 'query': {'field': 'geospatial_bounds', 'index': 'resources_index', 'vertical_bounds': {'from': 1.0, 'to': 2.0}, 'cmpop': 'overlaps'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result1), 2)
query_str = "{'and': [], 'or': [], 'query': {'field': 'geospatial_bounds', 'index': 'resources_index', 'vertical_bounds': {'from': 110.0, 'to': 120.0}, 'cmpop': 'contains'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=True)
self.assertEquals(len(result1), 1)
self.assertEquals(res_by_alias["DP3"], result1[0])
query_str = "{'and': [], 'or': [], 'query': {'field': 'geospatial_bounds', 'index': 'resources_index', 'vertical_bounds': {'from': 5.0, 'to': 30.0}, 'cmpop': 'within'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=True)
self.assertEquals(len(result1), 1)
self.assertEquals(res_by_alias["DP4"], result1[0])
# ----------------------------------------------------
# Temporal search
search_string = "search 'nominal_datetime' timebounds from '%s' to '%s' from 'resources_index'" %('2013-03-12','2013-03-19')
result = self.discovery.parse(search_string, id_only=True)
self.assertEquals(len(result), 2)
for dp in ["DP1", "DP2"]:
self.assertIn(res_by_alias[dp], result)
search_string = "search 'nominal_datetime' timebounds from '%s' to '%s' from 'resources_index'" %('2013-03-12','2013-11-19')
result = self.discovery.parse(search_string, id_only=True)
self.assertEquals(len(result), 4)
for dp in ["DP1", "DP2", "DP3", "DP4"]:
self.assertIn(res_by_alias[dp], result)
search_string = "search 'nominal_datetime' timebounds from '%s' to '%s' from 'resources_index'" %('2013-03-12','2013-03-13')
result = self.discovery.parse(search_string, id_only=True)
self.assertEquals(len(result), 1)
for dp in ["DP1"]:
self.assertIn(res_by_alias[dp], result)
def test_event_search(self):
from interface.objects import ResourceOperatorEvent, ResourceCommandEvent
t0 = 136304640000
events = [
("RME1", ResourceCommandEvent(origin="O1", origin_type="OT1", sub_type="ST1", ts_created=str(t0))),
("RME2", ResourceCommandEvent(origin="O2", origin_type="OT1", sub_type="ST2", ts_created=str(t0+1))),
("RME3", ResourceCommandEvent(origin="O2", origin_type="OT2", sub_type="ST3", ts_created=str(t0+2))),
("RLE1", ResourceOperatorEvent(origin="O1", origin_type="OT3", sub_type="ST4", ts_created=str(t0+3))),
("RLE2", ResourceOperatorEvent(origin="O3", origin_type="OT3", sub_type="ST5", ts_created=str(t0+4))),
("RLE3", ResourceOperatorEvent(origin="O3", origin_type="OT2", sub_type="ST6", ts_created=str(t0+5))),
]
ev_by_alias = {}
for (alias, event) in events:
evid, _ = self.container.event_repository.put_event(event)
ev_by_alias[alias] = evid
# ----------------------------------------------------
search_string = "search 'origin' is 'O1' from 'events_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 2)
search_string = "search 'origin_type' is 'OT2' from 'events_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 2)
search_string = "search 'sub_type' is 'ST6' from 'events_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 1)
search_string = "search 'ts_created' values from 136304640000 to 136304640000 from 'events_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 1)
search_string = "search 'type_' is 'ResourceCommandEvent' from 'events_index' order by 'ts_created'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 3)
class DiscoveryQueryTest(IonIntegrationTestCase):
"""Tests discovery in a somewhat integration environment. Only a container and a DiscoveryService instance
but no r2deploy and process"""
def setUp(self):
self._start_container()
self.discovery = DiscoveryService()
self.discovery.container = self.container
self.discovery.on_start()
self.rr = self.container.resource_registry
def _geopt(self, x1, y1):
return GeospatialIndex(lat=float(x1), lon=float(y1))
def _geobb(self, x1, y1, x2=None, y2=None, z1=0.0, z2=None):
if x2 is None: x2 = x1
if y2 is None: y2 = y1
if z2 is None: z2 = z1
return GeospatialBounds(geospatial_latitude_limit_north=float(y2),
geospatial_latitude_limit_south=float(y1),
geospatial_longitude_limit_west=float(x1),
geospatial_longitude_limit_east=float(x2),
geospatial_vertical_min=float(z1),
geospatial_vertical_max=float(z2))
def _temprng(self, t1="", t2=None):
if t2 is None: t2 = t1
return TemporalBounds(start_datetime=str(t1), end_datetime=str(t2))
def _geodp(self, x1, y1, x2=None, y2=None, z1=0.0, z2=None, t1="", t2=None):
if x2 is None: x2 = x1
if y2 is None: y2 = y1
if z2 is None: z2 = z1
if t2 is None: t2 = t1
bounds = self._geobb(x1, y1, x2, y2, z1, z2)
attrs = dict(geospatial_point_center=GeoUtils.calc_geospatial_point_center(bounds),
geospatial_bounds=bounds,
nominal_datetime=self._temprng(t1, t2))
return attrs
def test_basic_searching(self):
t0 = 1363046400
hour = 60*24
day = 60*60*24
resources = [
("ID1", InstrumentDevice(name='sonobuoy1', firmware_version='A1')),
("ID2", InstrumentDevice(name='sonobuoy2', firmware_version='A2')),
("ID3", InstrumentDevice(name='sonobuoy3', firmware_version='A3')),
("DP1", DataProduct(name='testData1', **self._geodp(5, 5, 15, 15, 0, 100, t0, t0+day))),
("DP2", DataProduct(name='testData2', **self._geodp(25, 5, 35, 15, 0, 100, t0+hour+day, t0+2*day))),
("DP3", DataProduct(name='testData3', **self._geodp(30, 10, 40, 20, 50, 200, t0+100*day, t0+110*day))),
("DP4", DataProduct(name='testData4', **self._geodp(30, 5, 32, 10, 5, 20, t0+100*day, t0+110*day))),
]
res_by_alias = {}
for (alias, resource) in resources:
rid,_ = self.rr.create(resource)
res_by_alias[alias] = rid
# ----------------------------------------------------
# Resource attribute search
# Resource attribute equals
search_string = "search 'firmware_version' is 'A2' from 'resources_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 1)
self.assertIsInstance(result[0], InstrumentDevice)
self.assertTrue(result[0].name == 'sonobuoy2')
self.assertTrue(result[0].firmware_version == 'A2')
query_str = "{'and': [], 'or': [], 'query': {'field': 'firmware_version', 'index': 'resources_index', 'value': 'A2'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result), len(result1))
self.assertEquals(result, result1)
# Resource attribute match
search_string = "search 'firmware_version' is 'A*' from 'resources_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 3)
query_str = "{'and': [], 'or': [], 'query': {'field': 'firmware_version', 'index': 'resources_index', 'value': 'A*'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result), len(result1))
self.assertEquals(result, result1)
# Resource attribute match with limit
search_string = "search 'firmware_version' is 'A*' from 'resources_index' limit 2"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 2)
query_str = "{'and': [], 'limit': 2, 'or': [], 'query': {'field': 'firmware_version', 'index': 'resources_index', 'value': 'A*'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result), len(result1))
self.assertEquals(result, result1)
# Check data products
search_string = "search 'name' is 'testData*' from 'resources_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 4)
search_string = "search 'type_' is 'DataProduct' from 'resources_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 4)
# ----------------------------------------------------
# Geospatial search
# Geospatial search - query bbox fully overlaps
search_string = "search 'geospatial_point_center' geo box top-left lat 180 lon -180 bottom-right lat -180 lon 180 from 'resources_index'"
result = self.discovery.parse(search_string, id_only=True)
self.assertGreaterEqual(len(result), 4)
for dp in ["DP1", "DP2", "DP3", "DP4"]:
self.assertIn(res_by_alias[dp], result)
search_string = "search 'geospatial_point_center' geo box top-left lat 20 lon 0 bottom-right lat 0 lon 20 from 'resources_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 1)
# Note that in Discovery intermediate format top_left=x1,y2 and bottom_right=x2,y1 contrary to naming
query_str = "{'and': [], 'or': [], 'query': {'top_left': [0.0, 20.0], 'bottom_right': [20.0, 0.0], 'field': 'index_location', 'index': 'resources_index'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result), len(result1))
self.assertEquals(result, result1)
# Geospatial bbox operators - overlaps (this is the default and should be the same as above)
query_str = "{'and': [], 'or': [], 'query': {'top_left': [0.0, 20.0], 'bottom_right': [20.0, 0.0], 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'overlaps'}}"
query_obj = eval(query_str)
result2 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result1), len(result2))
self.assertEquals(result1, result2)
# Geospatial bbox operators - contains (the resource contains the query)
query_str = "{'and': [], 'or': [], 'query': {'top_left': [0.0, 20.0], 'bottom_right': [20.0, 0.0], 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'contains'}}"
query_obj = eval(query_str)
result3 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result3), 0)
query_str = "{'and': [], 'or': [], 'query': {'top_left': [8.0, 11.0], 'bottom_right': [12.0, 9.0], 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'contains'}}"
query_obj = eval(query_str)
result3 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result3), 1)
# Geospatial bbox operators - within (the resource with the query)
query_str = "{'and': [], 'or': [], 'query': {'top_left': [0.0, 20.0], 'bottom_right': [20.0, 0.0], 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'within'}}"
query_obj = eval(query_str)
result3 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result3), 1)
query_str = "{'and': [], 'or': [], 'query': {'top_left': [15.0, 5.0], 'bottom_right': [5.0, 15.0], 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'within'}}"
query_obj = eval(query_str)
result3 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result3), 1)
query_str = "{'and': [], 'or': [], 'query': {'top_left': [14.0, 5.0], 'bottom_right': [5.0, 15.0], 'field': 'geospatial_bounds', 'index': 'resources_index', 'cmpop': 'within'}}"
query_obj = eval(query_str)
result3 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result3), 0)
# Geospatial search - query bbox partial overlaps
search_string = "search 'geospatial_bounds' geo box top-left lat 11 lon 9 bottom-right lat 9 lon 11 from 'resources_index'"
result = self.discovery.parse(search_string, id_only=False)
self.assertEquals(len(result), 1)
# ----------------------------------------------------
# Vertical search
search_string = "search 'geospatial_bounds' vertical from 0 to 500 from 'resources_index'"
result = self.discovery.parse(search_string, id_only=True)
self.assertGreaterEqual(len(result), 4)
for dp in ["DP1", "DP2", "DP3", "DP4"]:
self.assertIn(res_by_alias[dp], result)
query_str = "{'and': [], 'or': [], 'query': {'field': 'geospatial_bounds', 'index': 'resources_index', 'vertical_bounds': {'from': 0.0, 'to': 500.0}, 'cmpop': 'overlaps'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result1), 4)
query_str = "{'and': [], 'or': [], 'query': {'field': 'geospatial_bounds', 'index': 'resources_index', 'vertical_bounds': {'from': 1.0, 'to': 2.0}, 'cmpop': 'overlaps'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=False)
self.assertEquals(len(result1), 2)
query_str = "{'and': [], 'or': [], 'query': {'field': 'geospatial_bounds', 'index': 'resources_index', 'vertical_bounds': {'from': 110.0, 'to': 120.0}, 'cmpop': 'contains'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=True)
self.assertEquals(len(result1), 1)
self.assertEquals(res_by_alias["DP3"], result1[0])
query_str = "{'and': [], 'or': [], 'query': {'field': 'geospatial_bounds', 'index': 'resources_index', 'vertical_bounds': {'from': 5.0, 'to': 30.0}, 'cmpop': 'within'}}"
query_obj = eval(query_str)
result1 = self.discovery.query(query_obj, id_only=True)
self.assertEquals(len(result1), 1)
self.assertEquals(res_by_alias["DP4"], result1[0])
# ----------------------------------------------------
# Temporal search
search_string = "search 'nominal_datetime' timebounds from '%s' to '%s' from 'resources_index'" %('2013-03-12','2013-03-19')
result = self.discovery.parse(search_string, id_only=True)
self.assertEquals(len(result), 2)
for dp in ["DP1", "DP2"]:
self.assertIn(res_by_alias[dp], result)
search_string = "search 'nominal_datetime' timebounds from '%s' to '%s' from 'resources_index'" %('2013-03-12','2013-11-19')
result = self.discovery.parse(search_string, id_only=True)
self.assertEquals(len(result), 4)
for dp in ["DP1", "DP2", "DP3", "DP4"]:
self.assertIn(res_by_alias[dp], result)
search_string = "search 'nominal_datetime' timebounds from '%s' to '%s' from 'resources_index'" %('2013-03-12','2013-03-13')
result = self.discovery.parse(search_string, id_only=True)
self.assertEquals(len(result), 1)
for dp in ["DP1"]:
self.assertIn(res_by_alias[dp], result)
