def create_view(self, view_name='', description='', fields=None, order=None, filters=''):
"""Creates a view which has the specified search fields, the order in which the search fields are presented
to a query and a term filter.
@param view_name Name of the view
@param description Simple descriptive sentence
@param fields Search fields
@param order List of fields to determine order of precendence in which the results are presented
@param filter Simple term filter
@param view_name str
@param description str
@param fields list
@param order list
@param filters str
@retval view_id str
"""
res, _ = self.clients.resource_registry.find_resources(name=view_name, id_only=True)
if len(res) > 0:
raise BadRequest('The view resource with name: %s, already exists.' % view_name)
#======================
# Arg Validations
#======================
validate_is_instance(fields,list, 'Specified fields must be a list.')
validate_true(len(fields)>0, 'Specfied fields must be a list.')
if order is not None:
validate_is_instance(order,list, 'Specified order must be a list of fields')
for field in order:
if not field in fields:
raise BadRequest('The specified ordering field was not part of the search fields.')
fields = set(fields) # Convert fields to a set for aggregation across the catalogs
#======================================================================================================
# Priorty Queue Index Matching
#======================================================================================================
pq = [] # Priority queue for matching
catalog_id = None
catalogs, _ = self.clients.resource_registry.find_resources(restype=RT.Catalog, id_only=False)
for catalog in catalogs:
if set(catalog.catalog_fields).issubset(fields):
index_num = len(self.clients.catalog_management.list_indexes(catalog._id))
heapq.heappush(pq, (index_num,catalog))
if pq:
weight, catalog = heapq.heappop(pq)
if weight < self.heuristic_cutoff:
catalog_id = catalog._id
if catalog_id is None:
catalog_id = self.clients.catalog_management.create_catalog('%s_catalog'% view_name, keywords=list(fields))
view_res = View(name=view_name, description=description)
view_res.order = order
view_res.filters = filters
view_id, _ = self.clients.resource_registry.create(view_res)
self.clients.resource_registry.create_association(subject=view_id, predicate=PRED.hasCatalog,object=catalog_id)
return view_id
def create_view(self, view_name='', description='', fields=None, order=None, filters=''):
"""Creates a view which has the specified search fields, the order in which the search fields are presented
to a query and a term filter.
@param view_name Name of the view
@param description Simple descriptive sentence
@param fields Search fields
@param order List of fields to determine order of precendence in which the results are presented
@param filter Simple term filter
@param view_name str
@param description str
@param fields list
@param order list
@param filters str
@retval view_id str
"""
res, _ = self.clients.resource_registry.find_resources(name=view_name, id_only=True)
if len(res) > 0:
raise BadRequest('The view resource with name: %s, already exists.' % view_name)
#======================
# Arg Validations
#======================
validate_is_instance(fields,list, 'Specified fields must be a list.')
validate_true(len(fields)>0, 'Specfied fields must be a list.')
if order is not None:
validate_is_instance(order,list, 'Specified order must be a list of fields')
for field in order:
if not field in fields:
raise BadRequest('The specified ordering field was not part of the search fields.')
fields = set(fields) # Convert fields to a set for aggregation across the catalogs
#======================================================================================================
# Priorty Queue Index Matching
#======================================================================================================
pq = [] # Priority queue for matching
catalog_id = None
catalogs, _ = self.clients.resource_registry.find_resources(restype=RT.Catalog, id_only=False)
for catalog in catalogs:
if set(catalog.catalog_fields).issubset(fields):
index_num = len(self.clients.catalog_management.list_indexes(catalog._id))
heapq.heappush(pq, (index_num,catalog))
if pq:
weight, catalog = heapq.heappop(pq)
if weight < self.heuristic_cutoff:
catalog_id = catalog._id
if catalog_id is None:
catalog_id = self.clients.catalog_management.create_catalog('%s_catalog'% view_name, keywords=list(fields))
view_res = View(name=view_name, description=description)
view_res.order = order
view_res.filters = filters
view_id, _ = self.clients.resource_registry.create(view_res)
self.clients.resource_registry.create_association(subject=view_id, predicate=PRED.hasCatalog,object=catalog_id)
return view_id
def get_builtin_view(self, view_name):
view_obj = View(name=view_name)
if view_name == "resources_index":
rq = ResourceQuery()
view_obj.view_definition = rq.get_query()
return view_obj
elif view_name == "events_index":
eq = EventQuery()
view_obj.view_definition = eq.get_query()
return view_obj
def get_builtin_view(self, view_name):
view_obj = View(name=view_name)
if view_name == "resources_index":
rq = ResourceQuery()
view_obj.view_definition = rq.get_query()
return view_obj
elif view_name == "events_index":
eq = EventQuery()
view_obj.view_definition = eq.get_query()
return view_obj
def test_view_request(self, mock_es):
self.call_count = 0
def cb(*args, **kwargs):
self.call_count +=1
return [1]
self.discovery.elasticsearch_host = ''
self.discovery.elasticsearch_port = ''
v = View()
setattr(v, '_id', 'a')
self.discovery.list_catalogs = Mock()
self.discovery.list_catalogs.return_value = [1,2]
retval = self.discovery._multi(cb,v)
self.assertTrue(retval == [1,1])
c = Catalog()
setattr(c, '_id', 'c')
self.cms_list_indexes.return_value = [1,2]
retval = self.discovery._multi(cb,c)
self.assertTrue(retval == [1,1])
retval = self.discovery._multi(cb,v, limit=1)
self.assertTrue(retval == [1])
retval = self.discovery._multi(cb,c, limit=1)
self.assertTrue(retval == [1])
self.discovery._multi = Mock()
self.rr_read.return_value = v
self.discovery._multi.return_value = 'test'
retval = self.discovery.query_term('blah', 'field', 'value')
self.assertTrue(retval == 'test')
def get_builtin_view(self, view_name):
view_obj = View(name=view_name)
if view_name == "resources_index":
rq = ResourceQuery()
view_obj.view_definition = rq.get_query()
return view_obj
elif view_name == "data_products_index":
rq = ResourceQuery()
rq.set_filter(rq.filter_type(["DataProduct", "DataProcess", "Deployment", "InstrumentDevice", "InstrumentModel",
"InstrumentAgentInstance", "InstrumentAgent", "PlatformDevice", "PlatformModel",
"PlatformAgentInstance", "PlatformAgent", "PlatformSite", "Observatory", "UserRole",
"Org", "Attachment", "ExternalDatasetAgent", "ExternalDatasetAgentInstance"]))
view_obj.view_definition = rq.get_query()
return view_obj
elif view_name == "events_index":
eq = EventQuery()
view_obj.view_definition = eq.get_query()
return view_obj
def test_query_view(self):
res_objs = [
(IonObject(RT.ActorIdentity, name="Act1"), ),
(IonObject(RT.ActorIdentity, name="Act2"), ),
(IonObject(RT.TestInstrument,
name="ID1",
lcstate=LCS.DEPLOYED,
firmware_version='A1'), "Act1"),
(IonObject(RT.TestInstrument,
name="ID2",
lcstate=LCS.INTEGRATED,
firmware_version='A2'), "Act2"),
(IonObject(RT.TestPlatform, name="PD1"), ),
(IonObject(RT.TestPlatform, name="PD2"), ),
(IonObject(RT.TestSite, name="Site1", lcstate=LCS.DEPLOYED), ),
]
assocs = [
("PD1", PRED.hasTestDevice, "ID1"),
("PD2", PRED.hasTestDevice, "ID2"),
]
res_by_name = create_dummy_resources(res_objs, assocs)
# ----------------------------------------------------
# Resource attribute search
rq = ResourceQuery()
rq.set_filter(rq.filter_type(RT.TestInstrument))
view_obj = View(name="All TestInstrument resources",
view_definition=rq.get_query())
view_id = self.discovery.create_view(view_obj)
# TEST: View by ID
result = self.discovery.query_view(view_id, id_only=False)
self.assertEquals(len(result), 2)
self.assertTrue(
all(True for ro in result if ro.type_ == RT.TestInstrument))
# TEST: View by Name
result = self.discovery.query_view(
view_name="All TestInstrument resources", id_only=False)
self.assertEquals(len(result), 2)
self.assertTrue(
all(True for ro in result if ro.type_ == RT.TestInstrument))
# TEST: View plus ext_query
rq = ResourceQuery()
rq.set_filter(rq.filter_name("ID1"))
result = self.discovery.query_view(view_id,
id_only=False,
ext_query=rq.get_query())
self.assertEquals(len(result), 1)
self.assertEquals(result[0].name, "ID1")
# TEST: View with params (anonymous)
rq = ResourceQuery()
rq.set_filter(
rq.filter_type(RT.TestInstrument),
rq.filter_attribute("firmware_version", "$(firmware_version)"))
view_obj = View(
name=
"TestInstrument resources with a specific firmware - parameterized",
view_definition=rq.get_query())
view_id = self.discovery.create_view(view_obj)
view_params = {"firmware_version": "A2"}
result = self.discovery.query_view(view_id,
id_only=False,
search_args=view_params)
self.assertEquals(len(result), 1)
self.assertEquals(result[0].name, "ID2")
# TEST: View with params (anonymous) - no values provided
result = self.discovery.query_view(view_id, id_only=False)
self.assertEquals(len(result), 0)
# View with params (with definitions and defaults)
view_param_def = [
CustomAttribute(name="firmware_version", type="str", default="A1")
]
view_obj = View(
name=
"TestInstrument resources with a specific firmware - parameterized with defaults",
view_definition=rq.get_query(),
view_parameters=view_param_def)
view_id = self.discovery.create_view(view_obj)
# TEST: Definition defaults
result = self.discovery.query_view(view_id, id_only=False)
self.assertEquals(len(result), 1)
self.assertEquals(result[0].name, "ID1")
# TEST: Parameterized values
result = self.discovery.query_view(view_id,
id_only=False,
search_args=view_params)
self.assertEquals(len(result), 1)
self.assertEquals(result[0].name, "ID2")
# TEST: Parameterized association query for resource owner
rq = ResourceQuery()
rq.set_filter(rq.filter_associated_from_object("$(owner)"))
view_obj = View(name="Resources owned by actor - parameterized",
view_definition=rq.get_query())
view_id = self.discovery.create_view(view_obj)
view_params = {"owner": res_by_name["Act2"]}
result = self.discovery.query_view(view_id,
id_only=False,
search_args=view_params)
self.assertEquals(len(result), 1)
self.assertEquals(result[0].name, "ID2")
# TEST: Parameterized association query for resource owner with parameter value
view_params = {"owner": res_by_name["Act2"], "query_info": True}
result = self.discovery.query_view(view_id,
id_only=False,
search_args=view_params)
self.assertEquals(len(result), 2)
self.assertEquals(result[0].name, "ID2")
self.assertIn("_query_info", result[1])
self.assertIn("statement_sql", result[1])
# TEST: Builtin views
rq = ResourceQuery()
rq.set_filter(rq.filter_type(RT.TestSite))
result = self.discovery.query_view(view_name="resources_index",
id_only=False,
ext_query=rq.get_query())
self.assertEquals(len(result), 1)
self.assertEquals(result[0].name, "Site1")
# --- Events setup
from interface.objects import ResourceOperatorEvent, ResourceCommandEvent
t0 = 136304640000
events = [
("RME1",
ResourceCommandEvent(origin="O1",
origin_type="OT1",
sub_type="ST1",
ts_created=str(t0))),
("RME2",
ResourceCommandEvent(origin="O2",
origin_type="OT1",
sub_type="ST2",
ts_created=str(t0 + 1))),
("RME3",
ResourceCommandEvent(origin="O2",
origin_type="OT2",
sub_type="ST3",
ts_created=str(t0 + 2))),
("RLE1",
ResourceOperatorEvent(origin="O1",
origin_type="OT3",
sub_type="ST4",
ts_created=str(t0 + 3))),
("RLE2",
ResourceOperatorEvent(origin="O3",
origin_type="OT3",
sub_type="ST5",
ts_created=str(t0 + 4))),
("RLE3",
ResourceOperatorEvent(origin="O3",
origin_type="OT2",
sub_type="ST6",
ts_created=str(t0 + 5))),
]
ev_by_alias = create_dummy_events(events)
# TEST: Event query with views
eq = EventQuery()
eq.set_filter(eq.filter_type(OT.ResourceCommandEvent))
view_obj = View(name="All ResourceCommandEvent events",
view_definition=eq.get_query())
view_id = self.discovery.create_view(view_obj)
result = self.discovery.query_view(view_id, id_only=False)
self.assertEquals(len(result), 3)
self.assertTrue(
all(True for eo in result if eo.type_ == OT.ResourceCommandEvent))
# TEST: Event query with views - stripped format
result = self.discovery.query_view(
view_id,
id_only=False,
search_args=dict(attribute_filter=["origin"]))
self.assertEquals(len(result), 3)
self.assertTrue(all(True for eo in result if isinstance(eo, dict)))
self.assertTrue(all(True for eo in result if "origin" in eo))
self.assertTrue(all(True for eo in result if len(eo) <= 4))
# TEST: Builtin views
eq = EventQuery()
eq.set_filter(eq.filter_type(OT.ResourceCommandEvent))
result = self.discovery.query_view(view_name="events_index",
id_only=False,
ext_query=eq.get_query())
self.assertEquals(len(result), 3)
