class TestIntDataProcessManagementService(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
#print 'started container'
# Establish endpoint with container
container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
#print 'got CC client'
container_client.start_rel_from_url('res/deploy/r2sa.yml')
print 'started services'
# Now create client to DataProcessManagementService
self.DPMSclient = DataProcessManagementServiceClient(node=self.container.node)
self.RRclient = ResourceRegistryServiceClient(node=self.container.node)
def test_createDataProcess(self):
# test creating a new data process
print 'Creating new data process definition'
dpd_obj = IonObject(RT.DataProcessDefinition,
name='DPD1',
description='some new dpd',
process_source='some_source_reference')
try:
dprocd_id = self.DPMSclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new data process definition: %s" %ex)
print 'new dpd_id = ', dprocd_id
print 'Creating new subscription'
s_obj = IonObject(RT.Subscription,
name='S1',
description='some new subscription')
try:
s_id, version = self.RRclient.create(s_obj)
except BadRequest as ex:
self.fail("failed to create new subscription: %s" %ex)
print 'new s_id = ', s_id
print 'Creating new data product'
dprod_obj = IonObject(RT.Subscription,
name='DProd1',
description='some new data product')
try:
dprod_id, version = self.RRclient.create(dprod_obj)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new dprod_id = ', dprod_id
print 'Creating new data process'
try:
dproc_id = self.DPMSclient.create_data_process(dprocd_id, s_id, dprod_id)
except (BadRequest, NotFound, Conflict) as ex:
self.fail("failed to create new data process definition: %s" %ex)
def test_create_event_process(self):
"""
Test creating an event process
"""
process_definition = ProcessDefinition(name='test')
process_definition.definition = ''
rrc = ResourceRegistryServiceClient(node = self.container.node)
process_definition_id = rrc.create(process_definition)
self.mock_rr_client.find_objects = Mock()
self.mock_rr_client.find_objects.return_value = ['stream_id_1'], 'obj_assoc_1'
# self.mock_pd_client.schedule_process = Mock()
# self.mock_pd_client.schedule_process.return_value = 'process_id'
self.mock_rr_client.create_association = mocksignature(self.mock_rr_client.create_association)
pid = self.event_management.create_event_process(process_definition_id=process_definition_id,
event_types=['type_1', 'type_2'],
sub_types=['subtype_1', 'subtype_2'],
origins=['or_1', 'or_2'],
origin_types=['t1', 't2'],
out_data_products={'conductivity': 'id1'}
)
class TestMarineFacilityManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2sa.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
print 'started services'
def test_just_the_setup(self):
return
def test_resources_associations(self):
"""
create one of each resource and association used by MFMS
to guard against problems in ion-definitions
"""
#raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41")
#stuff we control
logical_instrument_id, _ = self.RR.create(any_old(RT.LogicalInstrument))
logical_platform_id, _ = self.RR.create(any_old(RT.LogicalPlatform))
logical_platform2_id, _ = self.RR.create(any_old(RT.LogicalPlatform))
marine_facility_id, _ = self.RR.create(any_old(RT.MarineFacility))
site_id, _ = self.RR.create(any_old(RT.Site))
site2_id, _ = self.RR.create(any_old(RT.Site))
#stuff we associate to
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
#logical_instrument
self.RR.create_association(logical_instrument_id, PRED.hasAgent, instrument_agent_id)
#logical_platform
self.RR.create_association(logical_platform_id, PRED.hasPlatform, logical_platform2_id)
self.RR.create_association(logical_platform_id, PRED.hasInstrument, logical_instrument_id)
self.RR.create_association(logical_platform_id, PRED.hasAgent, platform_agent_id)
#marine_facility
self.RR.create_association(marine_facility_id, PRED.hasSite, site_id)
self.RR.create_association(marine_facility_id, PRED.hasPlatform, logical_platform_id)
#site
self.RR.create_association(site_id, PRED.hasSite, site2_id)
class TestResourceRegistryAttachments(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
print 'started services'
def test_resource_registry_blob_sanity(self):
resource_id, _ = self.RR.create(IonObject(RT.Resource, name="foo"))
MY_CONTENT = "the quick brown fox etc etc etc"
#save
att_id = self.RR.create_attachment(
resource_id,
IonObject(RT.Attachment,
name="test.txt",
content=MY_CONTENT,
content_type="text/plain",
keywords=["test1", "test2"],
attachment_type=AttachmentType.BLOB))
#load
attachment = self.RR.read_attachment(att_id)
self.assertEqual("test.txt", attachment.name)
self.assertEqual("text/plain", attachment.content_type)
self.assertIn("test1", attachment.keywords)
self.assertIn("test2", attachment.keywords)
#content has changed; it's base64-encoded from what we put in
self.assertEqual(MY_CONTENT, attachment.content)
class TestResourceRegistryAttachments(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
print 'started services'
def test_resource_registry_blob_sanity(self):
resource_id, _ = self.RR.create(IonObject(RT.Resource, name="foo"))
MY_CONTENT = "the quick brown fox etc etc etc"
#save
att_id = self.RR.create_attachment(resource_id, IonObject(RT.Attachment,
name="test.txt",
content=MY_CONTENT,
content_type="text/plain",
keywords=["test1", "test2"],
attachment_type=AttachmentType.BLOB))
#load
attachment = self.RR.read_attachment(att_id)
self.assertEqual("test.txt", attachment.name)
self.assertEqual("text/plain", attachment.content_type)
self.assertIn("test1", attachment.keywords)
self.assertIn("test2", attachment.keywords)
#content has changed; it's base64-encoded from what we put in
self.assertEqual(MY_CONTENT, attachment.content)
class TestTransformWorkerSubscriptions(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management_client = DatasetManagementServiceClient(
node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_multi_subscriptions(self):
self.dp_list = []
self.event1_verified = Event()
self.event2_verified = Event()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product_one',
description='input test stream one')
self.input_dp_one_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product_two',
description='input test stream two')
self.input_dp_two_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
#retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_one_id, PRED.hasStream, RT.Stream, True)
self.stream_one_id = stream_ids[0]
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_two_id, PRED.hasStream, RT.Stream, True)
self.stream_two_id = stream_ids[0]
dpd_id = self.create_data_process_definition()
dp1_func_output_dp_id, dp2_func_output_dp_id = self.create_output_data_products(
)
first_dp_id = self.create_data_process_one(dpd_id,
dp1_func_output_dp_id)
second_dp_id = self.create_data_process_two(dpd_id,
self.input_dp_two_id,
dp2_func_output_dp_id)
#retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=first_dp_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_transform_worker subscription_obj: %s',
subscription_objs[0])
#create subscription to stream ONE, create data process and publish granule on stream ONE
#create a queue to catch the published granules of stream ONE
self.subscription_one_id = self.pubsub_client.create_subscription(
name='parsed_subscription_one',
stream_ids=[self.stream_one_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_one_id)
self.pubsub_client.activate_subscription(self.subscription_one_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_one_id)
stream_route_one = self.pubsub_client.read_stream_route(
self.stream_one_id)
self.publisher_one = StandaloneStreamPublisher(
stream_id=self.stream_one_id, stream_route=stream_route_one)
self.start_event_listener()
#data process 1 adds conductivity + pressure and puts the result in salinity
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher_one.publish(msg=rdt.to_granule(),
stream_id=self.stream_one_id)
#retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=second_dp_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_transform_worker subscription_obj: %s',
subscription_objs[0])
#create subscription to stream ONE and TWO, move TW subscription, create data process and publish granule on stream TWO
#create a queue to catch the published granules of stream TWO
self.subscription_two_id = self.pubsub_client.create_subscription(
name='parsed_subscription_one_two',
stream_ids=[self.stream_two_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_two_id)
self.pubsub_client.activate_subscription(self.subscription_two_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_two_id)
stream_route_two = self.pubsub_client.read_stream_route(
self.stream_two_id)
self.publisher_two = StandaloneStreamPublisher(
stream_id=self.stream_two_id, stream_route=stream_route_two)
#data process 1 adds conductivity + pressure and puts the result in salinity
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher_one.publish(msg=rdt.to_granule(),
stream_id=self.stream_one_id)
#data process 2 adds salinity + pressure and puts the result in conductivity
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [22]
rdt['pressure'] = [4]
rdt['salinity'] = [1]
self.publisher_two.publish(msg=rdt.to_granule(),
stream_id=self.stream_two_id)
self.assertTrue(self.event2_verified.wait(self.wait_time))
self.assertTrue(self.event1_verified.wait(self.wait_time))
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_two_transforms_inline(self):
self.dp_list = []
self.event1_verified = Event()
self.event2_verified = Event()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product_one',
description='input test stream one')
self.input_dp_one_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
dpd_id = self.create_data_process_definition()
dp1_func_output_dp_id, dp2_func_output_dp_id = self.create_output_data_products(
)
first_dp_id = self.create_data_process_one(dpd_id,
dp1_func_output_dp_id)
second_dp_id = self.create_data_process_two(dpd_id,
dp1_func_output_dp_id,
dp2_func_output_dp_id)
#retrieve subscription from data process one
subscription_objs, _ = self.rrclient.find_objects(
subject=first_dp_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_transform_worker subscription_obj: %s',
subscription_objs[0])
#retrieve the Stream for these data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_one_id, PRED.hasStream, RT.Stream, True)
self.stream_one_id = stream_ids[0]
#the input to data process two is the output from data process one
stream_ids, assoc_ids = self.rrclient.find_objects(
dp1_func_output_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_two_id = stream_ids[0]
# Run provenance on the output dataproduct of the second data process to see all the links
# are as expected
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
dp2_func_output_dp_id)
# Do a basic check to see if there were 2 entries in the provenance graph. Parent and Child.
self.assertTrue(len(output_data_product_provenance) == 3)
# confirm that the linking from the output dataproduct to input dataproduct is correct
self.assertTrue(
dp1_func_output_dp_id in
output_data_product_provenance[dp2_func_output_dp_id]['parents'])
self.assertTrue(
self.input_dp_one_id in
output_data_product_provenance[dp1_func_output_dp_id]['parents'])
#create subscription to stream ONE, create data process and publish granule on stream ONE
#create a queue to catch the published granules of stream ONE
subscription_id = self.pubsub_client.create_subscription(
name='parsed_subscription',
stream_ids=[self.stream_one_id, self.stream_two_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
subscription_id)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
subscription_id)
stream_route_one = self.pubsub_client.read_stream_route(
self.stream_one_id)
self.publisher_one = StandaloneStreamPublisher(
stream_id=self.stream_one_id, stream_route=stream_route_one)
#retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=second_dp_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_transform_worker subscription_obj: %s',
subscription_objs[0])
#data process 1 adds conductivity + pressure and puts the result in salinity
#data process 2 adds salinity + pressure and puts the result in conductivity
self.start_event_listener()
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher_one.publish(msg=rdt.to_granule(),
stream_id=self.stream_one_id)
self.assertTrue(self.event2_verified.wait(self.wait_time))
self.assertTrue(self.event1_verified.wait(self.wait_time))
def create_data_process_definition(self):
#two data processes using one transform and one DPD
# Set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri=
'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id, add_array_dpd_id, binding='add_array_func')
return add_array_dpd_id
def create_data_process_one(self, data_process_definition_id,
output_dataproduct):
# Create the data process
#data process 1 adds conductivity + pressure and puts the result in salinity
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "salinity"
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=data_process_definition_id,
inputs=[self.input_dp_one_id],
outputs=[output_dataproduct],
argument_map=argument_map,
out_param_name=output_param)
self.damsclient.register_process(dp1_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process,
dp1_data_process_id)
self.dp_list.append(dp1_data_process_id)
return dp1_data_process_id
def create_data_process_two(self, data_process_definition_id,
input_dataproduct, output_dataproduct):
# Create the data process
#data process 2 adds salinity + pressure and puts the result in conductivity
argument_map = {'arr1': 'salinity', 'arr2': 'pressure'}
output_param = 'conductivity'
dp2_func_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=data_process_definition_id,
inputs=[input_dataproduct],
outputs=[output_dataproduct],
argument_map=argument_map,
out_param_name=output_param)
self.damsclient.register_process(dp2_func_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process,
dp2_func_data_process_id)
self.dp_list.append(dp2_func_data_process_id)
return dp2_func_data_process_id
def create_output_data_products(self):
dp1_outgoing_stream_id = self.pubsub_client.create_stream_definition(
name='dp1_stream', parameter_dictionary_id=self.parameter_dict_id)
dp1_output_dp_obj = IonObject(RT.DataProduct,
name='data_process1_data_product',
description='output of add array func')
dp1_func_output_dp_id = self.dataproductclient.create_data_product(
dp1_output_dp_obj, dp1_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product,
dp1_func_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp1_func_output_dp_id,
PRED.hasStream, None, True)
self._output_stream_one_id = stream_ids[0]
dp2_func_outgoing_stream_id = self.pubsub_client.create_stream_definition(
name='dp2_stream', parameter_dictionary_id=self.parameter_dict_id)
dp2_func_output_dp_obj = IonObject(
RT.DataProduct,
name='data_process2_data_product',
description='output of add array func')
dp2_func_output_dp_id = self.dataproductclient.create_data_product(
dp2_func_output_dp_obj, dp2_func_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product,
dp2_func_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp2_func_output_dp_id,
PRED.hasStream, None, True)
self._output_stream_two_id = stream_ids[0]
subscription_id = self.pubsub_client.create_subscription(
'validator',
data_product_ids=[dp1_func_output_dp_id, dp2_func_output_dp_id])
self.addCleanup(self.pubsub_client.delete_subscription,
subscription_id)
def on_granule(msg, route, stream_id):
log.debug('recv_packet stream_id: %s route: %s msg: %s',
stream_id, route, msg)
self.validate_output_granule(msg, route, stream_id)
validator = StandaloneStreamSubscriber('validator',
callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
subscription_id)
return dp1_func_output_dp_id, dp2_func_output_dp_id
def validate_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
data_process_event = args[0]
log.debug("DataProcessStatusEvent: %s",
str(data_process_event.__dict__))
#if data process already created, check origin
if not 'data process assigned to transform worker' in data_process_event.description:
self.assertIn(data_process_event.origin, self.dp_list)
def validate_output_granule(self, msg, route, stream_id):
self.assertTrue(
stream_id in
[self._output_stream_one_id, self._output_stream_two_id])
rdt = RecordDictionaryTool.load_from_granule(msg)
log.debug('validate_output_granule stream_id: %s', stream_id)
if stream_id == self._output_stream_one_id:
sal_val = rdt['salinity']
log.debug('validate_output_granule sal_val: %s', sal_val)
np.testing.assert_array_equal(sal_val, np.array([3]))
self.event1_verified.set()
else:
cond_val = rdt['conductivity']
log.debug('validate_output_granule cond_val: %s', cond_val)
np.testing.assert_array_equal(cond_val, np.array([5]))
self.event2_verified.set()
def start_event_listener(self):
es = EventSubscriber(event_type=OT.DataProcessStatusEvent,
callback=self.validate_event)
es.start()
self.addCleanup(es.stop)
class DiscoveryIntTest(IonIntegrationTestCase):
def setUp(self):
super(DiscoveryIntTest, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(
host=es_host,
port=es_port,
timeout=10
)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete,index)
IndexManagementService._es_call(es.index_delete,index)
def poll(self, tries, callback, *args, **kwargs):
'''
Polling wrapper for queries
Elasticsearch may not index and cache the changes right away so we may need
a couple of tries and a little time to go by before the results show.
'''
for i in xrange(tries):
retval = callback(*args, **kwargs)
if retval:
return retval
time.sleep(0.2)
return None
def test_traversal(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results = self.discovery.traverse(dp_id)
results.sort()
correct = [pd_id, transform_id]
correct.sort()
self.assertTrue(results == correct, '%s' % results)
def test_iterative_traversal(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results = self.discovery.iterative_traverse(dp_id)
results.sort()
correct = [transform_id]
self.assertTrue(results == correct)
results = self.discovery.iterative_traverse(dp_id, 1)
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@skipIf(not use_es, 'No ElasticSearch')
def test_view_crud(self):
view_id = self.discovery.create_view('big_view',fields=['name'])
catalog_id = self.discovery.list_catalogs(view_id)[0]
index_ids = self.catalog.list_indexes(catalog_id)
self.assertTrue(len(index_ids))
view = self.discovery.read_view(view_id)
self.assertIsInstance(view,View)
self.assertTrue(view.name == 'big_view')
view.name = 'not_so_big_view'
self.discovery.update_view(view)
view = self.discovery.read_view(view_id)
self.assertTrue(view.name == 'not_so_big_view')
self.discovery.delete_view(view_id)
with self.assertRaises(NotFound):
self.discovery.read_view(view_id)
def test_view_best_match(self):
#---------------------------------------------------------------
# Matches the best catalog available OR creates a new one
#---------------------------------------------------------------
catalog_id = self.catalog.create_catalog('dev', keywords=['name','model'])
view_id = self.discovery.create_view('exact_view', fields=['name','model'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids == [catalog_id])
view_id = self.discovery.create_view('another_view', fields=['name','model'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids == [catalog_id])
view_id = self.discovery.create_view('big_view', fields=['name'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids != [catalog_id])
@skipIf(not use_es, 'No ElasticSearch')
def test_basic_searching(self):
#- - - - - - - - - - - - - - - - -
# set up the fake resources
#- - - - - - - - - - - - - - - - -
instrument_pool = [
InstrumentDevice(name='sonobuoy1', firmware_version='1'),
InstrumentDevice(name='sonobuoy2', firmware_version='2'),
InstrumentDevice(name='sonobuoy3', firmware_version='3')
]
for instrument in instrument_pool:
self.rr.create(instrument)
view_id = self.discovery.create_view('devices', fields=['firmware_version'])
search_string = "search 'firmware_version' is '2' from '%s'"%view_id
results = self.poll(5, self.discovery.parse,search_string)
result = results[0]['_source']
self.assertIsInstance(result, InstrumentDevice)
self.assertTrue(result.name == 'sonobuoy2')
self.assertTrue(result.firmware_version == '2')
@skipIf(not use_es, 'No ElasticSearch')
def test_associative_searching(self):
dp_id,_ = self.rr.create(DataProduct('test_foo'))
ds_id,_ = self.rr.create(Dataset('test_bar', registered=True))
self.rr.create_association(subject=dp_id, object=ds_id, predicate='hasDataset')
search_string = "search 'type_' is 'Dataset' from 'resources_index' and belongs to '%s'" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(ds_id in results)
def test_iterative_associative_searching(self):
#--------------------------------------------------------------------------------
# Tests the ability to limit the iterations
#--------------------------------------------------------------------------------
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
search_string = "belongs to '%s' depth 1" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
correct = [transform_id]
self.assertTrue(results == correct, '%s' % results)
search_string = "belongs to '%s' depth 2" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@skipIf(not use_es, 'No ElasticSearch')
def test_ranged_value_searching(self):
discovery = self.discovery
rr = self.rr
view_id = discovery.create_view('bank_view', fields=['cash_balance'])
bank_id, _ = rr.create(BankAccount(name='broke', cash_balance=10))
search_string = "search 'cash_balance' values from 0 to 100 from '%s'" % view_id
results = self.poll(5, discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == bank_id)
bank_id, _ = rr.create(BankAccount(name='broke', cash_balance=90))
search_string = "search 'cash_balance' values from 80 from '%s'" % view_id
results = self.poll(5, discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == bank_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_collections_searching(self):
site_id, _ = self.rr.create(Site(name='black_mesa'))
view_id = self.discovery.create_view('big', fields=['name'])
# Add the site to a new collection
collection_id = self.ims.create_collection('resource_collection', [site_id])
search_string = "search 'name' is '*' from '%s' and in '%s'" %(view_id, collection_id)
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0] == site_id, '%s' % results)
@skipIf(not use_es, 'No ElasticSearch')
def test_search_by_name(self):
inst_dev = InstrumentDevice(name='test_dev',serial_number='ABC123')
dev_id, _ = self.rr.create(inst_dev)
self.discovery.create_view('devs',fields=['name','serial_number'])
search_string = "search 'serial_number' is 'abc*' from 'devs'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dev_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_search_by_name_index(self):
inst_dev = InstrumentDevice(name='test_dev',serial_number='ABC123')
dev_id, _ = self.rr.create(inst_dev)
search_string = "search 'serial_number' is 'abc*' from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dev_id)
bank_acc = BankAccount(name='blah', cash_balance=10)
res_id , _ = self.rr.create(bank_acc)
search_string = "search 'cash_balance' values from 0 to 100 from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == res_id)
#@skipIf(not use_es, 'No ElasticSearch')
@skip('Skip until time to refactor, data_format is removed from DataProduct resource')
def test_data_product_search(self):
# Create the dataproduct
dp = DataProduct(name='test_product')
dp.data_format.name = 'test_signal'
dp.data_format.description = 'test signal'
dp.data_format.character_set = 'utf8'
dp.data_format.nominal_sampling_rate_maximum = '44000'
dp.data_format.nominal_sampling_rate_minimum = '44000'
dp.CDM_data_type = 'basic'
dp_id, _ = self.rr.create(dp)
search_string = "search 'data_format.name' is 'test_signal' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'CDM_data_type' is 'basic' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'data_format.character_set' is 'utf8' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_events_search(self):
# Create a resource to force a new event
dp = DataProcess()
dp_id, rev = self.rr.create(dp)
search_string = "SEARCH 'origin' IS '%s' FROM 'events_index'" % dp_id
results = self.poll(9, self.discovery.parse,search_string)
origin_type = results[0]['_source'].origin_type
origin_id = results[0]['_source'].origin
self.assertTrue(origin_type == RT.DataProcess)
self.assertTrue(origin_id == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_geo_distance_search(self):
pd = PlatformDevice(name='test_dev')
pd_id, _ = self.rr.create(pd)
search_string = "search 'index_location' geo distance 20 km from lat 0 lon 0 from 'devices_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == pd_id)
self.assertTrue(results[0]['_source'].name == 'test_dev')
@skipIf(not use_es, 'No ElasticSearch')
def test_geo_bbox_search(self):
pd = PlatformDevice(name='test_dev')
pd.index_location.lat = 5
pd.index_location.lon = 5
pd_id, _ = self.rr.create(pd)
search_string = "search 'index_location' geo box top-left lat 10 lon 0 bottom-right lat 0 lon 10 from 'devices_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == pd_id)
self.assertTrue(results[0]['_source'].name == 'test_dev')
@skipIf(not use_es, 'No ElasticSearch')
def test_time_search(self):
today = date.today()
yesterday = today - timedelta(days=1)
tomorrow = today + timedelta(days=1)
data_product = DataProduct()
dp_id, _ = self.rr.create(data_product)
search_string = "search 'ts_created' time from '%s' to '%s' from 'data_products_index'" % (yesterday, tomorrow)
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results,'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'ts_created' time from '%s' from 'data_products_index'" % yesterday
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results,'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_user_search(self):
user = UserInfo()
user.name = 'test'
user.contact.phones.append('5551212')
user_id, _ = self.rr.create(user)
search_string = 'search "name" is "test" from "users_index"'
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == user_id)
self.assertTrue(results[0]['_source'].name == 'test')
search_string = 'search "contact.phones" is "5551212" from "users_index"'
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == user_id)
self.assertTrue(results[0]['_source'].name == 'test')
@skipIf(not use_es, 'No ElasticSearch')
def test_subobject_search(self):
contact = ContactInformation()
contact.email = '*****@*****.**'
contact.individual_name_family = 'Tester'
contact.individual_names_given = 'Intern'
dp = DataProduct(name='example')
dp.contacts.append(contact)
dp_id,_ = self.rr.create(dp)
#--------------------------------------------------------------------------------
# Example using the full field name
#--------------------------------------------------------------------------------
search_string = 'search "contacts.email" is "*****@*****.**" from "data_products"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
self.assertEquals(results[0]['_source'].name, 'example')
#--------------------------------------------------------------------------------
# Example using a sub-object's field name (ambiguous searching)
#--------------------------------------------------------------------------------
search_string = 'search "individual_names_given" is "Intern" from "data_products"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
self.assertEquals(results[0]['_source'].name, 'example')
@skipIf(not use_es, 'No ElasticSearch')
def test_descriptive_phrase_search(self):
dp = DataProduct(name='example', description='This is simply a description for this data product')
dp_id, _ = self.rr.create(dp)
search_string = 'search "description" like "description for" from "data_products"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
self.assertEquals(results[0]['_source'].name, 'example')
@skipIf(not use_es, 'No ElasticSearch')
def test_ownership_searching(self):
# Create two data products so that there is competition to the search, one is parsed
# (with conductivity as a parameter) and the other is raw
dp = DataProduct(name='example dataproduct')
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id)
tdom, sdom = time_series_domain()
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
dp_id = self.data_product_management.create_data_product(dp, stream_definition_id=stream_def_id, exchange_point='xp1')
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict')
stream_def_id = self.pubsub_management.create_stream_definition('ctd raw', parameter_dictionary_id=pdict_id)
dp = DataProduct(name='WRONG')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
self.data_product_management.create_data_product(dp, stream_definition_id=stream_def_id, exchange_point='xp1')
parameter_search = 'search "name" is "conductivity" from "resources_index"'
results = self.poll(9, self.discovery.parse, parameter_search)
param_id = results[0]['_id']
data_product_search = 'search "name" is "*" from "data_products_index" and has "%s"' % param_id
results = self.poll(9, self.discovery.parse, data_product_search)
print results
self.assertEquals(results[0], dp_id)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.dataset_management = self.datasetclient
def test_createDataProcess(self):
#---------------------------------------------------------------------------
# Data Process Definition
#---------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# Make assertion on the newly registered data process definition
data_process_definition = self.rrclient.read(dprocdef_id)
self.assertEquals(data_process_definition.name, 'ctd_L0_all')
self.assertEquals(data_process_definition.description, 'transform ctd package into three separate L0 streams')
self.assertEquals(data_process_definition.module, 'ion.processes.data.transforms.ctd.ctd_L0_all')
self.assertEquals(data_process_definition.class_name, 'ctd_L0_all')
# Read the data process definition using data process management and make assertions
dprocdef_obj = self.dataprocessclient.read_data_process_definition(dprocdef_id)
self.assertEquals(dprocdef_obj.class_name,'ctd_L0_all')
self.assertEquals(dprocdef_obj.module,'ion.processes.data.transforms.ctd.ctd_L0_all')
#---------------------------------------------------------------------------
# Create an input instrument
#---------------------------------------------------------------------------
instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product')
instrument_id, rev = self.rrclient.create(instrument_obj)
# Register the instrument so that the data producer and stream object are created
data_producer_id = self.damsclient.register_instrument(instrument_id)
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_input_stream_definition_to_data_process_definition(ctd_stream_def_id, dprocdef_id )
# Assert that the link between the stream definition and the data process definition was done
assocs = self.rrclient.find_associations(subject=dprocdef_id, predicate=PRED.hasInputStreamDefinition, object=ctd_stream_def_id, id_only=True)
self.assertIsNotNone(assocs)
#---------------------------------------------------------------------------
# Input Data Product
#---------------------------------------------------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
input_dp_obj = IonObject( RT.DataProduct,
name='InputDataProduct',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=ctd_stream_def_id, exchange_point='test')
#Make assertions on the input data product created
input_dp_obj = self.rrclient.read(input_dp_id)
self.assertEquals(input_dp_obj.name, 'InputDataProduct')
self.assertEquals(input_dp_obj.description, 'some new dp')
self.damsclient.assign_data_product(instrument_id, input_dp_id)
# Retrieve the stream via the DataProduct->Stream associations
stream_ids, _ = self.rrclient.find_objects(input_dp_id, PRED.hasStream, None, True)
self.in_stream_id = stream_ids[0]
#---------------------------------------------------------------------------
# Output Data Product
#---------------------------------------------------------------------------
outgoing_stream_conductivity_id = self.pubsubclient.create_stream_definition(name='conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_conductivity_id, dprocdef_id,binding='conductivity' )
outgoing_stream_pressure_id = self.pubsubclient.create_stream_definition(name='pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_pressure_id, dprocdef_id, binding='pressure' )
outgoing_stream_temperature_id = self.pubsubclient.create_stream_definition(name='temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_temperature_id, dprocdef_id, binding='temperature' )
self.output_products={}
output_dp_obj = IonObject(RT.DataProduct,
name='conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_1 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_conductivity_id)
self.output_products['conductivity'] = output_dp_id_1
output_dp_obj = IonObject(RT.DataProduct,
name='pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_2 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_pressure_id)
self.output_products['pressure'] = output_dp_id_2
output_dp_obj = IonObject(RT.DataProduct,
name='temperature',
description='transform output ',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_3 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_temperature_id)
self.output_products['temperature'] = output_dp_id_3
#---------------------------------------------------------------------------
# Create the data process
#---------------------------------------------------------------------------
def _create_data_process():
dproc_id = self.dataprocessclient.create_data_process(dprocdef_id, [input_dp_id], self.output_products)
return dproc_id
dproc_id = _create_data_process()
# Make assertions on the data process created
data_process = self.dataprocessclient.read_data_process(dproc_id)
# Assert that the data process has a process id attached
self.assertIsNotNone(data_process.process_id)
# Assert that the data process got the input data product's subscription id attached as its own input_susbcription_id attribute
self.assertIsNotNone(data_process.input_subscription_id)
output_data_product_ids = self.rrclient.find_objects(subject=dproc_id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct, id_only=True)
self.assertEquals(Set(output_data_product_ids[0]), Set([output_dp_id_1,output_dp_id_2,output_dp_id_3]))
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}})
def test_createDataProcessUsingSim(self):
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" )
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver" )
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
port_agent_config = {
'device_addr': 'sbe37-simulator.oceanobservatories.org',
'device_port': 4001,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'command_port': 4002,
'data_port': 4003,
'log_level': 5,
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", svr_addr="localhost",
comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port,
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE32_CDM', parameter_dictionary_id=pdict_id)
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id)
dp_obj.name = 'ctd_raw'
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
self.output_products={}
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#-------------------------------
# Create listener for data process events and verify that events are received.
#-------------------------------
# todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events
#todo (contd) ... I believe the capability does not exist yet now. ANS And SA are not yet publishing any workflow life cycle events (Swarbhanu)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
#-------------------------------
# Retrieve a list of all data process defintions in RR and validate that the DPD is listed
#-------------------------------
# todo: add this validate for Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions
# todo: This capability is not yet completed (Swarbhanu)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
#todo: check that activate event is received L4-CI-SA-RQ-367
#todo... (it looks like no event is being published when the data process is activated... so below, we just check for now
# todo... that the subscription is indeed activated) (Swarbhanu)
# todo: monitor process to see if it is active (sa-rq-182)
ctd_l0_all_data_process = self.rrclient.read(ctd_l0_all_data_process_id)
input_subscription_id = ctd_l0_all_data_process.input_subscription_id
subs = self.rrclient.read(input_subscription_id)
self.assertTrue(subs.activated)
# todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the extended resources for data process definition and for data process
#-------------------------------
extended_process_definition = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
self.assertEqual(1, len(extended_process_definition.data_processes))
log.debug("test_createDataProcess: extended_process_definition %s", str(extended_process_definition))
extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id)
self.assertEqual(1, len(extended_process.input_data_products))
log.debug("test_createDataProcess: extended_process %s", str(extended_process))
#-------------------------------
# Cleanup
#-------------------------------
self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id)
self.dataprocessclient.delete_data_process_definition(ctd_L0_all_dprocdef_id)
self.dataprocessclient.force_delete_data_process(ctd_l0_all_data_process_id)
self.dataprocessclient.force_delete_data_process_definition(ctd_L0_all_dprocdef_id)
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_notification(self, user_name = '', instrument_id='', product_id=''):
#--------------------------------------------------------------------------------------
# Make notification request objects
#--------------------------------------------------------------------------------------
notification_request_1 = NotificationRequest( name= 'notification_1',
origin=instrument_id,
origin_type="instrument",
event_type='ResourceLifecycleEvent')
notification_request_2 = NotificationRequest( name='notification_2',
origin=product_id,
origin_type="data product",
event_type='DetectionEvent')
#--------------------------------------------------------------------------------------
# Create a user and get the user_id
#--------------------------------------------------------------------------------------
user = UserInfo()
user.name = user_name
user.contact.email = '*****@*****.**' % user_name
user_id, _ = self.rrclient.create(user)
#--------------------------------------------------------------------------------------
# Create notification
#--------------------------------------------------------------------------------------
self.usernotificationclient.create_notification(notification=notification_request_1, user_id=user_id)
self.usernotificationclient.create_notification(notification=notification_request_2, user_id=user_id)
log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id) )
return user_id
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def _check_computed_attributes_of_extended_instrument(self, expected_instrument_device_id = '',extended_instrument = None):
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
# the following assert will not work without elasticsearch.
#self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
# Verify the computed attribute for user notification requests
self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
notifications = extended_instrument.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(expected_instrument_device_id, notification.origin)
self.assertEqual("instrument", notification.origin_type)
self.assertEqual('ResourceLifecycleEvent', notification.event_type)
def _check_computed_attributes_of_extended_product(self, expected_data_product_id = '', extended_data_product = None):
self.assertEqual(expected_data_product_id, extended_data_product._id)
log.debug("extended_data_product.computed: %s", extended_data_product.computed)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_data_product.computed.product_download_size_estimated, ComputedFloatValue)
self.assertIsInstance(extended_data_product.computed.number_active_subscriptions, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue)
self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.user_notification_requests, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.active_user_subscriptions, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.past_user_subscriptions, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_contents_updated, ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue)
# exact text here keeps changing to fit UI capabilities. keep assertion general...
self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value) )
notifications = extended_data_product.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(expected_data_product_id, notification.origin)
self.assertEqual("data product", notification.origin_type)
self.assertEqual('DetectionEvent', notification.event_type)
@attr('LOCOINT')
#@unittest.skip('refactoring')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 90}}})
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw')
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict')
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations = [raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) " , instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config,
alerts= [])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s' , data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s' , dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0] )
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# setup notifications for the device and parsed data product
user_id_1 = self._create_notification( user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1)
#---------- Create notifications for another user and verify that we see different computed subscriptions for the two users ---------
user_id_2 = self._create_notification( user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug('Data product streams2 = %s' , str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id2 = %s' , dataset_ids[0])
self.raw_dataset = dataset_ids[0]
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
#log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s" , str(self._ia_client))
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s" , str(retval))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.INACTIVE, state)
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s" , str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.IDLE, state)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s" , str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s" , str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.STOPPED, state)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.IDLE, state)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
for i in xrange(10):
monitor = DatasetMonitor(dataset_id=self.parsed_dataset)
self._ia_client.execute_resource(AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE))
if not monitor.wait():
raise AssertionError('Failed on the %ith granule' % i)
monitor.stop()
# cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
# for i in xrange(10):
# retval = self._ia_client.execute_resource(cmd)
# log.debug("test_activateInstrumentSample: return from sample %s" , str(retval))
log.debug( "test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s" , str(reply))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data_raw, Granule)
rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw)
log.debug("RDT raw: %s", str(rdt_raw.pretty_print()) )
self.assertIn('raw', rdt_raw)
raw_vals = rdt_raw['raw']
all_raw = "".join(raw_vals)
# look for 't' entered after a prompt -- ">t"
t_commands = all_raw.count(">t")
if 10 != t_commands:
log.error("%s raw_vals: ", len(raw_vals))
for i, r in enumerate(raw_vals): log.error("raw val %s: %s", i, [r])
self.fail("Expected 10 't' strings in raw_vals, got %s" % t_commands)
else:
log.debug("%s raw_vals: ", len(raw_vals))
for i, r in enumerate(raw_vals): log.debug("raw val %s: %s", i, [r])
replay_data_parsed = self.dataretrieverclient.retrieve(self.parsed_dataset)
self.assertIsInstance(replay_data_parsed, Granule)
rdt_parsed = RecordDictionaryTool.load_from_granule(replay_data_parsed)
log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt_parsed.pretty_print()) )
self.assertIn('temp', rdt_parsed)
temp_vals = rdt_parsed['temp']
pressure_vals = rdt_parsed['pressure']
if 10 != len(temp_vals):
log.error("%s temp_vals: %s", len(temp_vals), temp_vals)
self.fail("Expected 10 temp_vals, got %s" % len(temp_vals))
log.debug("l4-ci-sa-rq-138")
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
#--------------------------------------------------------------------------------
# Get the extended data product to see if it contains the granules
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id1, user_id=user_id_1)
def poller(extended_product):
return len(extended_product.computed.user_notification_requests.value) == 1
poll(poller, extended_product, timeout=30)
self._check_computed_attributes_of_extended_product( expected_data_product_id = data_product_id1, extended_data_product = extended_product)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_1)
#--------------------------------------------------------------------------------
# For the second user, check the extended data product and the extended intrument
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id2, user_id=user_id_2)
self._check_computed_attributes_of_extended_product(expected_data_product_id = data_product_id2, extended_data_product = extended_product)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_2)
self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument)
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(data_product_id1)
self.dpclient.delete_data_product(data_product_id2)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.dataset_management = self.datasetclient
self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node)
def test_createDataProcess(self):
#---------------------------------------------------------------------------
# Data Process Definition
#---------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# Make assertion on the newly registered data process definition
data_process_definition = self.rrclient.read(dprocdef_id)
self.assertEquals(data_process_definition.name, 'ctd_L0_all')
self.assertEquals(data_process_definition.description, 'transform ctd package into three separate L0 streams')
self.assertEquals(data_process_definition.module, 'ion.processes.data.transforms.ctd.ctd_L0_all')
self.assertEquals(data_process_definition.class_name, 'ctd_L0_all')
# Read the data process definition using data process management and make assertions
dprocdef_obj = self.dataprocessclient.read_data_process_definition(dprocdef_id)
self.assertEquals(dprocdef_obj.class_name,'ctd_L0_all')
self.assertEquals(dprocdef_obj.module,'ion.processes.data.transforms.ctd.ctd_L0_all')
#---------------------------------------------------------------------------
# Create an input instrument
#---------------------------------------------------------------------------
instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product')
instrument_id, rev = self.rrclient.create(instrument_obj)
# Register the instrument so that the data producer and stream object are created
data_producer_id = self.damsclient.register_instrument(instrument_id)
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_input_stream_definition_to_data_process_definition(ctd_stream_def_id, dprocdef_id )
# Assert that the link between the stream definition and the data process definition was done
assocs = self.rrclient.find_associations(subject=dprocdef_id, predicate=PRED.hasInputStreamDefinition, object=ctd_stream_def_id, id_only=True)
self.assertIsNotNone(assocs)
#---------------------------------------------------------------------------
# Input Data Product
#---------------------------------------------------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
input_dp_obj = IonObject( RT.DataProduct,
name='InputDataProduct',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=ctd_stream_def_id, exchange_point='test')
#Make assertions on the input data product created
input_dp_obj = self.rrclient.read(input_dp_id)
self.assertEquals(input_dp_obj.name, 'InputDataProduct')
self.assertEquals(input_dp_obj.description, 'some new dp')
self.damsclient.assign_data_product(instrument_id, input_dp_id)
# Retrieve the stream via the DataProduct->Stream associations
stream_ids, _ = self.rrclient.find_objects(input_dp_id, PRED.hasStream, None, True)
self.in_stream_id = stream_ids[0]
#---------------------------------------------------------------------------
# Output Data Product
#---------------------------------------------------------------------------
outgoing_stream_conductivity_id = self.pubsubclient.create_stream_definition(name='conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_conductivity_id, dprocdef_id,binding='conductivity' )
outgoing_stream_pressure_id = self.pubsubclient.create_stream_definition(name='pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_pressure_id, dprocdef_id, binding='pressure' )
outgoing_stream_temperature_id = self.pubsubclient.create_stream_definition(name='temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_temperature_id, dprocdef_id, binding='temperature' )
self.output_products={}
output_dp_obj = IonObject(RT.DataProduct,
name='conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_1 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_conductivity_id)
self.output_products['conductivity'] = output_dp_id_1
output_dp_obj = IonObject(RT.DataProduct,
name='pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_2 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_pressure_id)
self.output_products['pressure'] = output_dp_id_2
output_dp_obj = IonObject(RT.DataProduct,
name='temperature',
description='transform output ',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_3 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_temperature_id)
self.output_products['temperature'] = output_dp_id_3
#---------------------------------------------------------------------------
# Create the data process
#---------------------------------------------------------------------------
def _create_data_process():
dproc_id = self.dataprocessclient.create_data_process(dprocdef_id, [input_dp_id], self.output_products)
return dproc_id
dproc_id = _create_data_process()
# Make assertions on the data process created
data_process = self.dataprocessclient.read_data_process(dproc_id)
# Assert that the data process has a process id attached
self.assertIsNotNone(data_process.process_id)
# Assert that the data process got the input data product's subscription id attached as its own input_susbcription_id attribute
self.assertIsNotNone(data_process.input_subscription_id)
output_data_product_ids = self.rrclient.find_objects(subject=dproc_id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct, id_only=True)
self.assertEquals(Set(output_data_product_ids[0]), Set([output_dp_id_1,output_dp_id_2,output_dp_id_3]))
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}})
def test_createDataProcessUsingSim(self):
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" )
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg")
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
port_agent_config = {
'device_addr': 'sbe37-simulator.oceanobservatories.org',
'device_port': 4001,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'command_port': 4002,
'data_port': 4003,
'log_level': 5,
}
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE32_CDM', parameter_dictionary_id=pdict_id)
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id)
dp_obj.name = 'ctd_raw'
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
self.output_products={}
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#-------------------------------
# Create listener for data process events and verify that events are received.
#-------------------------------
# todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events
#todo (contd) ... I believe the capability does not exist yet now. ANS And SA are not yet publishing any workflow life cycle events (Swarbhanu)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
data_process = self.rrclient.read(ctd_l0_all_data_process_id)
process_id = data_process.process_id
self.addCleanup(self.process_dispatcher.cancel_process, process_id)
#-------------------------------
# Wait until the process launched in the create_data_process() method is actually running, before proceeding further in this test
#-------------------------------
gate = ProcessStateGate(self.process_dispatcher.read_process, process_id, ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The data process (%s) did not spawn in 30 seconds" % process_id)
#-------------------------------
# Retrieve a list of all data process defintions in RR and validate that the DPD is listed
#-------------------------------
# todo: Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions
# todo: data topics are being handled by pub sub at the level of streams
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
#todo: check that activate event is received L4-CI-SA-RQ-367
#todo... (it looks like no event is being published when the data process is activated... so below, we just check for now
# todo... that the subscription is indeed activated) (Swarbhanu)
# todo: monitor process to see if it is active (sa-rq-182)
ctd_l0_all_data_process = self.rrclient.read(ctd_l0_all_data_process_id)
input_subscription_id = ctd_l0_all_data_process.input_subscription_id
subs = self.rrclient.read(input_subscription_id)
self.assertTrue(subs.activated)
# todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the extended resources for data process definition and for data process
#-------------------------------
extended_process_definition = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
self.assertEqual(1, len(extended_process_definition.data_processes))
log.debug("test_createDataProcess: extended_process_definition %s", str(extended_process_definition))
extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id)
self.assertEqual(1, len(extended_process.input_data_products))
log.debug("test_createDataProcess: extended_process %s", str(extended_process))
################################ Test the removal of data processes ##################################
#-------------------------------------------------------------------
# Test the deleting of the data process
#-------------------------------------------------------------------
# Before deleting, get the input streams, output streams and the subscriptions so that they can be checked after deleting
# dp_obj_1 = self.rrclient.read(ctd_l0_all_data_process_id)
# input_subscription_id = dp_obj_1.input_subscription_id
# out_prods, _ = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True)
# in_prods, _ = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasInputProduct, id_only=True)
# in_streams = []
# for in_prod in in_prods:
# streams, _ = self.rrclient.find_objects(in_prod, PRED.hasStream, id_only=True)
# in_streams.extend(streams)
# out_streams = []
# for out_prod in out_prods:
# streams, _ = self.rrclient.find_objects(out_prod, PRED.hasStream, id_only=True)
# out_streams.extend(streams)
# Deleting the data process
self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id)
# Check that the data process got removed. Check the lcs state. It should be retired
dp_obj = self.rrclient.read(ctd_l0_all_data_process_id)
self.assertEquals(dp_obj.lcstate, LCS.RETIRED)
# Check for process defs still attached to the data process
dpd_assn_ids = self.rrclient.find_associations(subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcessDefinition, id_only=True)
self.assertEquals(len(dpd_assn_ids), 0)
# Check for output data product still attached to the data process
out_products, assocs = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True)
self.assertEquals(len(out_products), 0)
self.assertEquals(len(assocs), 0)
# Check for input data products still attached to the data process
inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct)
self.assertEquals(len(inprod_associations), 0)
# Check for input data products still attached to the data process
inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct)
self.assertEquals(len(inprod_associations), 0)
# Check of the data process has been deactivated
self.assertIsNone(dp_obj.input_subscription_id)
# Read the original subscription id of the data process and check that it has been deactivated
with self.assertRaises(NotFound):
self.pubsubclient.read_subscription(input_subscription_id)
#-------------------------------------------------------------------
# Delete the data process definition
#-------------------------------------------------------------------
# before deleting, get the process definition being associated to in order to be able to check later if the latter gets deleted as it should
proc_def_ids, proc_def_asocs = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasProcessDefinition)
self.dataprocessclient.delete_data_process_definition(ctd_L0_all_dprocdef_id)
# check that the data process definition has been retired
dp_proc_def = self.rrclient.read(ctd_L0_all_dprocdef_id)
self.assertEquals(dp_proc_def.lcstate, LCS.RETIRED)
# Check for old associations of this data process definition
proc_defs, proc_def_asocs = self.rrclient.find_objects(ctd_L0_all_dprocdef_id, PRED.hasProcessDefinition)
self.assertEquals(len(proc_defs), 0)
# find all associations where this is the subject
_, obj_assns = self.rrclient.find_objects(subject= ctd_L0_all_dprocdef_id, id_only=True)
self.assertEquals(len(obj_assns), 0)
################################ Test the removal of data processes ##################################
# Try force delete... This should simply delete the associations and the data process object
# from the resource registry
#---------------------------------------------------------------------------------------------------------------
# Force deleting a data process
#---------------------------------------------------------------------------------------------------------------
self.dataprocessclient.force_delete_data_process(ctd_l0_all_data_process_id)
# find all associations where this is the subject
_, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True)
# find all associations where this is the object
_, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True)
self.assertEquals(len(obj_assns), 0)
self.assertEquals(len(sbj_assns), 0)
with self.assertRaises(NotFound):
self.rrclient.read(ctd_l0_all_data_process_id)
#---------------------------------------------------------------------------------------------------------------
# Force deleting a data process definition
#---------------------------------------------------------------------------------------------------------------
self.dataprocessclient.force_delete_data_process_definition(ctd_L0_all_dprocdef_id)
# find all associations where this is the subject
_, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True)
# find all associations where this is the object
_, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True)
self.assertEquals(len(obj_assns), 0)
self.assertEquals(len(sbj_assns), 0)
with self.assertRaises(NotFound):
self.rrclient.read(ctd_l0_all_data_process_id)
class TestResourceRegistryAttachments(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
print 'started services'
def test_resource_registry_blob_sanity(self):
resource_id, _ = self.RR.create(IonObject(RT.Resource, name="foo"))
MY_CONTENT = "the quick brown fox etc etc etc"
#save
att_id = self.RR.create_attachment(
resource_id,
IonObject(RT.Attachment,
name="test.txt",
content=MY_CONTENT,
content_type="text/plain",
keywords=["test1", "test2"],
attachment_type=AttachmentType.BLOB))
#load
attachment = self.RR.read_attachment(att_id, include_content=True)
self.assertEqual("test.txt", attachment.name)
self.assertEqual("text/plain", attachment.content_type)
self.assertIn("test1", attachment.keywords)
self.assertIn("test2", attachment.keywords)
#content has changed; it's base64-encoded from what we put in
self.assertEqual(MY_CONTENT, attachment.content)
obj = self.RR.read(resource_id)
self.assertEqual(obj.name, "foo")
obj.name = "TheDudeAbides"
obj = self.RR.update(obj)
obj = self.RR.read(resource_id)
self.assertEqual(obj.name, "TheDudeAbides")
att = self.RR.find_attachments(resource_id)
self.assertNotEqual(att, None)
actor_identity_obj = IonObject("ActorIdentity", name="name")
actor_identity_obj_id, actor_identity_obj_rev = self.RR.create(
actor_identity_obj)
user_info_obj = IonObject("UserInfo", name="name")
user_info_obj_id, user_info_obj_rev = self.RR.create(user_info_obj)
assoc_id, assoc_rev = self.RR.create_association(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
self.assertNotEqual(assoc_id, None)
find_assoc = self.RR.find_associations(actor_identity_obj_id,
PRED.hasInfo, user_info_obj_id)
self.assertTrue(find_assoc[0]._id == assoc_id)
subj = self.RR.find_subjects(RT.ActorIdentity, PRED.hasInfo,
user_info_obj_id, True)
res_obj1 = self.RR.read_object(actor_identity_obj_id, PRED.hasInfo,
RT.UserInfo)
self.assertEquals(res_obj1._id, user_info_obj_id)
self.RR.delete_association(assoc_id)
self.RR.delete_attachment(att_id)
self.RR.delete(resource_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
unittest # suppress an pycharm inspector error if all unittest.skip references are commented out
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(
any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
instrument_site_id, _ = self.RR.create(any_old(RT.InstrumentSite))
platform_agent_instance_id, _ = self.RR.create(
any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_site_id, _ = self.RR.create(any_old(RT.PlatformSite))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(instrument_agent_instance_id,
PRED.hasAgentDefinition,
instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance,
instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer,
data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice,
sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource,
instrument_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_agent_instance_id,
PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance,
platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice,
instrument_device_id)
self.RR.create_association(instrument_site_id, PRED.hasDevice,
instrument_device_id)
self.RR.create_association(platform_site_id, PRED.hasDevice,
platform_device_id)
self.RR.create_association(platform_site_id, PRED.hasSite,
instrument_site_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel,
sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice,
instrument_device_id)
sensor_model_id #is only a target
#create a parsed product for this instrument output
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain=tdom,
spatial_domain=sdom)
pdict_id = self.DSC.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(
instrument_device_id,
"/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(
instrument_device_id,
"/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
extended_instrument = self.IMS.get_instrument_device_extension(
instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id,
instrument_model_id)
# Lifecycle
self.assertEquals(len(extended_instrument.lcstate_transitions), 5)
self.assertEquals(
set(extended_instrument.lcstate_transitions.keys()),
set(['develop', 'deploy', 'retire', 'plan', 'integrate']))
self.assertEquals(len(extended_instrument.availability_transitions), 2)
self.assertEquals(
set(extended_instrument.availability_transitions.keys()),
set(['enable', 'announce']))
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(
extended_instrument.computed.last_data_received_datetime,
ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
self.assertIsInstance(
extended_instrument.computed.power_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.communications_status_roll_up,
ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.location_status_roll_up,
ComputedIntValue)
log.debug("extended_instrument.computed: %s",
extended_instrument.computed)
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name,
extended_instrument.instrument_model.name)
#check agent instance
inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id)
self.assertEqual(inst_agent_instance_obj.name,
extended_instrument.agent_instance.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
#compound assoc return list of lists so check the first element
self.assertEqual(inst_agent_obj.name,
extended_instrument.instrument_agent.name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name,
extended_instrument.platform_device.name)
extended_platform = self.IMS.get_platform_device_extension(
platform_device_id)
self.assertEqual(1, len(extended_platform.portals))
self.assertEqual(1, len(extended_platform.portal_instruments))
#self.assertEqual(1, len(extended_platform.computed.portal_status.value)) # no agent started so NO statuses reported
self.assertEqual(1, len(extended_platform.instrument_devices))
self.assertEqual(instrument_device_id,
extended_platform.instrument_devices[0]._id)
self.assertEqual(1, len(extended_platform.instrument_models))
self.assertEqual(instrument_model_id,
extended_platform.instrument_models[0]._id)
self.assertEquals(extended_platform.platform_agent._id,
platform_agent_id)
self.assertEquals(len(extended_platform.lcstate_transitions), 5)
self.assertEquals(
set(extended_platform.lcstate_transitions.keys()),
set(['develop', 'deploy', 'retire', 'plan', 'integrate']))
self.assertEquals(len(extended_platform.availability_transitions), 2)
self.assertEquals(
set(extended_platform.availability_transitions.keys()),
set(['enable', 'announce']))
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
#check data_product_parameters_set
self.assertEqual(
ComputedValueAvailability.PROVIDED,
extended_instrument.computed.data_product_parameters_set.status)
self.assertTrue('Parsed_Canonical' in extended_instrument.computed.
data_product_parameters_set.value)
# the ctd parameters should include 'temp'
self.assertTrue('temp' in extended_instrument.computed.
data_product_parameters_set.value['Parsed_Canonical'])
#none of these will work because there is no agent
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.firmware_version.status)
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.operational_state.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.power_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.communications_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.data_status_roll_up.status)
# self.assertEqual(DeviceStatusType.STATUS_OK,
# extended_instrument.computed.data_status_roll_up.value)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.location_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.recent_events.status)
# self.assertEqual([], extended_instrument.computed.recent_events.value)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.RR2.pluck(sensor_device_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_platform_agent_instance(
platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.OMS.force_delete_instrument_site(instrument_site_id)
self.OMS.force_delete_platform_site(platform_site_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
def test_custom_attributes(self):
"""
Test assignment of custom attributes
"""
instModel_obj = IonObject(OT.CustomAttribute,
name='SBE37IMModelAttr',
description="model custom attr")
instrument_model_id, _ = self.RR.create(
any_old(RT.InstrumentModel,
{"custom_attributes": [instModel_obj]}))
instrument_device_id, _ = self.RR.create(
any_old(
RT.InstrumentDevice, {
"custom_attributes": {
"favorite_color": "red",
"bogus_attr": "should raise warning"
}
}))
self.IMS.assign_instrument_model_to_instrument_device(
instrument_model_id, instrument_device_id)
# cleanup
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
def _get_datastore(self, dataset_id):
dataset = self.DSC.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_data_producer(self):
idevice_id = self.IMS.create_instrument_device(
any_old(RT.InstrumentDevice))
self.assertEqual(
1,
len(
self.RR2.
find_data_producer_ids_of_instrument_device_using_has_data_producer(
idevice_id)))
pdevice_id = self.IMS.create_platform_device(any_old(
RT.PlatformDevice))
self.assertEqual(
1,
len(
self.RR2.
find_data_producer_ids_of_platform_device_using_has_data_producer(
pdevice_id)))
@attr('PREP')
def test_prepare_resource_support(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(
any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_agent_instance_id, _ = self.RR.create(
any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
instrument_device2_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
instrument_device3_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_device2_id, _ = self.RR.create(any_old(RT.PlatformDevice))
sensor_device2_id, _ = self.RR.create(any_old(RT.SensorDevice))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(instrument_agent_instance_id,
PRED.hasAgentDefinition,
instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance,
instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer,
data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice,
sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource,
instrument_device_id)
self.RR.create_association(instrument_device2_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(org_id, PRED.hasResource,
instrument_device2_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_agent_instance_id,
PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance,
platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice,
instrument_device_id)
self.RR.create_association(platform_device2_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_device2_id, PRED.hasDevice,
instrument_device2_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel,
sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice,
instrument_device_id)
self.RR.create_association(sensor_device2_id, PRED.hasModel,
sensor_model_id)
self.RR.create_association(sensor_device2_id, PRED.hasDevice,
instrument_device2_id)
sensor_model_id #is only a target
#set lcstate - used for testing prepare - not setting all to DEVELOP, only some
self.RR.execute_lifecycle_transition(instrument_agent_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(instrument_device_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(instrument_device2_id,
LCE.DEVELOP)
self.RR.execute_lifecycle_transition(platform_device_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(platform_device2_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(platform_agent_id, LCE.DEVELOP)
#create a parsed product for this instrument output
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain=tdom,
spatial_domain=sdom)
pdict_id = self.DSC.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(
instrument_device_id,
"/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(
instrument_device_id,
"/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
def ion_object_encoder(obj):
return obj.__dict__
#First call to create
instrument_data = self.IMS.prepare_instrument_device_support()
#print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_data._id, '')
self.assertEqual(instrument_data.type_,
OT.InstrumentDevicePrepareSupport)
self.assertEqual(
len(instrument_data.associations['InstrumentModel'].resources), 1)
self.assertEqual(
instrument_data.associations['InstrumentModel'].resources[0]._id,
instrument_model_id)
self.assertEqual(
len(instrument_data.associations['InstrumentAgentInstance'].
resources), 1)
self.assertEqual(
instrument_data.associations['InstrumentAgentInstance'].
resources[0]._id, instrument_agent_instance_id)
self.assertEqual(
len(instrument_data.associations['InstrumentModel'].
associated_resources), 0)
self.assertEqual(
len(instrument_data.associations['InstrumentAgentInstance'].
associated_resources), 0)
self.assertEqual(
len(instrument_data.associations['SensorDevice'].resources), 0)
#Next call to update
instrument_data = self.IMS.prepare_instrument_device_support(
instrument_device_id)
#print 'Update results'
#print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_data._id, instrument_device_id)
self.assertEqual(instrument_data.type_,
OT.InstrumentDevicePrepareSupport)
self.assertEqual(
len(instrument_data.associations['InstrumentModel'].resources), 1)
self.assertEqual(
instrument_data.associations['InstrumentModel'].resources[0]._id,
instrument_model_id)
self.assertEqual(
len(instrument_data.associations['InstrumentAgentInstance'].
resources), 1)
self.assertEqual(
instrument_data.associations['InstrumentAgentInstance'].
resources[0]._id, instrument_agent_instance_id)
self.assertEqual(
len(instrument_data.associations['InstrumentModel'].
associated_resources), 1)
self.assertEqual(
instrument_data.associations['InstrumentModel'].
associated_resources[0].s, instrument_device_id)
self.assertEqual(
instrument_data.associations['InstrumentModel'].
associated_resources[0].o, instrument_model_id)
self.assertEqual(
len(instrument_data.associations['InstrumentAgentInstance'].
associated_resources), 1)
self.assertEqual(
instrument_data.associations['InstrumentAgentInstance'].
associated_resources[0].o, instrument_agent_instance_id)
self.assertEqual(
instrument_data.associations['InstrumentAgentInstance'].
associated_resources[0].s, instrument_device_id)
self.assertEqual(
len(instrument_data.associations['SensorDevice'].resources), 1)
self.assertEqual(
instrument_data.associations['SensorDevice'].resources[0]._id,
sensor_device_id)
self.assertEqual(
len(instrument_data.associations['SensorDevice'].
associated_resources), 1)
self.assertEqual(
instrument_data.associations['SensorDevice'].
associated_resources[0].o, instrument_device_id)
self.assertEqual(
instrument_data.associations['SensorDevice'].
associated_resources[0].s, sensor_device_id)
self.assertEqual(
instrument_data.associations['InstrumentModel'].assign_request.
request_parameters['instrument_device_id'], instrument_device_id)
#test prepare for create of instrument agent instance
instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support(
)
#print 'Update results'
#print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_agent_data._id, '')
self.assertEqual(instrument_agent_data.type_,
OT.InstrumentAgentInstancePrepareSupport)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentDevice'].
resources), 2)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentAgent'].resources
), 1)
self.assertEqual(
instrument_agent_data.associations['InstrumentAgent'].resources[0].
_id, instrument_agent_id)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentDevice'].
associated_resources), 0)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentAgent'].
associated_resources), 0)
#test prepare for update of instrument agent instance to see if it is associated with the instrument that was created
instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support(
instrument_agent_instance_id=instrument_agent_instance_id)
#print 'Update results'
#print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_agent_data._id,
instrument_agent_instance_id)
self.assertEqual(instrument_agent_data.type_,
OT.InstrumentAgentInstancePrepareSupport)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentDevice'].
resources), 3)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentAgent'].resources
), 1)
self.assertEqual(
instrument_agent_data.associations['InstrumentAgent'].resources[0].
_id, instrument_agent_id)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentDevice'].
associated_resources), 1)
self.assertEqual(
instrument_agent_data.associations['InstrumentDevice'].
associated_resources[0].s, instrument_device_id)
self.assertEqual(
instrument_agent_data.associations['InstrumentDevice'].
associated_resources[0].o, instrument_agent_instance_id)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentAgent'].
associated_resources), 1)
self.assertEqual(
instrument_agent_data.associations['InstrumentAgent'].
associated_resources[0].o, instrument_agent_id)
self.assertEqual(
instrument_agent_data.associations['InstrumentAgent'].
associated_resources[0].s, instrument_agent_instance_id)
self.assertEqual(
instrument_agent_data.associations['InstrumentAgent'].
assign_request.request_parameters['instrument_agent_instance_id'],
instrument_agent_instance_id)
#test prepare for update of data product to see if it is associated with the instrument that was created
data_product_data = self.DP.prepare_data_product_support(
data_product_id1)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent=2)
self.assertEqual(data_product_data._id, data_product_id1)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].resources),
1)
self.assertEqual(
len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].
associated_resources), 1)
self.assertEqual(
data_product_data.associations['StreamDefinition'].
associated_resources[0].s, data_product_id1)
self.assertEqual(
len(data_product_data.associations['Dataset'].associated_resources
), 0)
self.assertEqual(
len(data_product_data.
associations['InstrumentDeviceHasOutputProduct'].resources), 3)
self.assertEqual(
len(data_product_data.associations[
'InstrumentDeviceHasOutputProduct'].associated_resources), 1)
self.assertEqual(
data_product_data.associations['InstrumentDeviceHasOutputProduct'].
associated_resources[0].s, instrument_device_id)
self.assertEqual(
data_product_data.associations['InstrumentDeviceHasOutputProduct'].
associated_resources[0].o, data_product_id1)
self.assertEqual(
len(data_product_data.associations['PlatformDevice'].resources), 2)
platform_data = self.IMS.prepare_platform_device_support()
#print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2)
self.assertEqual(platform_data._id, '')
self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport)
self.assertEqual(
len(platform_data.associations['PlatformModel'].resources), 1)
self.assertEqual(
platform_data.associations['PlatformModel'].resources[0]._id,
platform_model_id)
self.assertEqual(
len(platform_data.associations['PlatformAgentInstance'].resources),
1)
self.assertEqual(
platform_data.associations['PlatformAgentInstance'].resources[0].
_id, platform_agent_instance_id)
self.assertEqual(
len(platform_data.associations['PlatformModel'].
associated_resources), 0)
self.assertEqual(
len(platform_data.associations['PlatformAgentInstance'].
associated_resources), 0)
self.assertEqual(
len(platform_data.associations['InstrumentDevice'].resources), 1)
platform_data = self.IMS.prepare_platform_device_support(
platform_device_id)
#print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2)
self.assertEqual(platform_data._id, platform_device_id)
self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport)
self.assertEqual(
len(platform_data.associations['PlatformModel'].resources), 1)
self.assertEqual(
platform_data.associations['PlatformModel'].resources[0]._id,
platform_model_id)
self.assertEqual(
len(platform_data.associations['PlatformAgentInstance'].resources),
1)
self.assertEqual(
platform_data.associations['PlatformAgentInstance'].resources[0].
_id, platform_agent_instance_id)
self.assertEqual(
len(platform_data.associations['PlatformModel'].
associated_resources), 1)
self.assertEqual(
platform_data.associations['PlatformModel'].
associated_resources[0].s, platform_device_id)
self.assertEqual(
platform_data.associations['PlatformModel'].
associated_resources[0].o, platform_model_id)
self.assertEqual(
len(platform_data.associations['PlatformAgentInstance'].
associated_resources), 1)
self.assertEqual(
platform_data.associations['PlatformAgentInstance'].
associated_resources[0].o, platform_agent_instance_id)
self.assertEqual(
platform_data.associations['PlatformAgentInstance'].
associated_resources[0].s, platform_device_id)
self.assertEqual(
len(platform_data.associations['InstrumentDevice'].resources), 2)
#self.assertEqual(len(platform_data.associations['InstrumentDevice'].associated_resources), 1)
#self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].s, platform_device_id)
#self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].o, instrument_device_id)
self.assertEqual(
platform_data.associations['PlatformModel'].assign_request.
request_parameters['platform_device_id'], platform_device_id)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.RR2.pluck(sensor_device_id)
self.RR2.pluck(sensor_device2_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_device(instrument_device2_id)
self.IMS.force_delete_platform_agent_instance(
platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_device(platform_device2_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_device(sensor_device2_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
class CatalogManagementIntTest(IonIntegrationTestCase):
def setUp(self):
super(CatalogManagementIntTest,self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.cms_cli = CatalogManagementServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
def test_create_catalog(self):
#-------------------------------------
# Tests basic catalog creation
#-------------------------------------
index1 = Index(name='index1')
index1.options.attribute_match = ['common']
index_id, _ = self.rr_cli.create(index1)
catalog_id = self.cms_cli.create_catalog('common_catalog', ['common'])
self.assertTrue(index_id in self.cms_cli.list_indexes(catalog_id))
def test_create_catalog_matching(self):
#-------------------------------------------
# Tests catalog creation and index matching
#-------------------------------------------
indexes = [
Index(name='index1'),
Index(name='index2'),
Index(name='index3')
]
indexes[0].options = SearchOptions(**{ # Devices
'attribute_match' : ['name','model','serial'],
'geo_fields' : ['nominal_location']
})
indexes[1].options = SearchOptions(**{ # BankAccount
'attribute_match' : ['name','description','owner'],
'range_fields' : ['cash_balance']
})
indexes[2].options = SearchOptions(**{ # Hybrid
'attribute_match' : ['name'],
'range_fields' : ['arc'],
'geo_fields' : ['nominal_location']
})
index_ids = list([ self.rr_cli.create(i)[0] for i in indexes])
devices_catalog_id = self.cms_cli.create_catalog('devices_catalog', ['model'])
self.assertTrue(index_ids[0] in self.cms_cli.list_indexes(devices_catalog_id), 'The catalog did\'nt match the correct index.')
accounts_catalog_id = self.cms_cli.create_catalog('accounts_catalog', ['cash_balance'])
self.assertTrue(index_ids[1] in self.cms_cli.list_indexes(accounts_catalog_id), 'The catalog did\'nt match the correct index.')
geo_catalog_id = self.cms_cli.create_catalog('geo_catalog', ['nominal_location'])
self.assertTrue(index_ids[2] in self.cms_cli.list_indexes(geo_catalog_id), 'The catalog did\'nt match the correct index.')
names_catalog_id = self.cms_cli.create_catalog('names_catalog', ['name'])
names_list = self.cms_cli.list_indexes(names_catalog_id)
self.assertTrue(set(index_ids) == set(names_list), 'The catalog did\'nt match the correct index.')
def test_catalog_field_exclusion(self):
half_match = Index(name='half_match', options=SearchOptions(attribute_match=['one']))
full_match = Index(name='full_match', options=SearchOptions(attribute_match=['one','two']))
half_match_id, _ = self.rr_cli.create(half_match)
full_match_id, _ = self.rr_cli.create(full_match)
catalog_id = self.cms_cli.create_catalog('test_cat', keywords=['one','two'])
index_list = self.cms_cli.list_indexes(catalog_id, id_only=True)
self.assertTrue(index_list == [full_match_id])
def test_update_catalog(self):
empty_catalog_id = self.cms_cli.create_catalog('empty_catalog')
empty_catalog = self.rr_cli.read(empty_catalog_id)
empty_catalog.description = 'testing update'
self.cms_cli.update_catalog(empty_catalog)
empty_catalog = self.rr_cli.read(empty_catalog_id)
self.assertTrue(empty_catalog.description == 'testing update')
def test_read_catalog(self):
catalog_res = Catalog(name='fake_catalog')
catalog_id, _ = self.rr_cli.create(catalog_res)
cat = self.cms_cli.read_catalog(catalog_id)
self.assertTrue(cat.name == 'fake_catalog')
def test_delete_catalog(self):
empty_catalog_id, _ = self.rr_cli.create(Catalog(name='empty_catalog'))
self.rr_cli.read(empty_catalog_id) # Throws exception if it doesn't exist
self.cms_cli.delete_catalog(empty_catalog_id)
with self.assertRaises(NotFound):
self.rr_cli.read(empty_catalog_id)
with self.assertRaises(NotFound):
self.cms_cli.read_catalog(empty_catalog_id)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.dataset_management = self.datasetclient
self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node)
def test_createDataProcess(self):
#---------------------------------------------------------------------------
# Data Process Definition
#---------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# Make assertion on the newly registered data process definition
data_process_definition = self.rrclient.read(dprocdef_id)
self.assertEquals(data_process_definition.name, 'ctd_L0_all')
self.assertEquals(data_process_definition.description, 'transform ctd package into three separate L0 streams')
self.assertEquals(data_process_definition.module, 'ion.processes.data.transforms.ctd.ctd_L0_all')
self.assertEquals(data_process_definition.class_name, 'ctd_L0_all')
# Read the data process definition using data process management and make assertions
dprocdef_obj = self.dataprocessclient.read_data_process_definition(dprocdef_id)
self.assertEquals(dprocdef_obj.class_name,'ctd_L0_all')
self.assertEquals(dprocdef_obj.module,'ion.processes.data.transforms.ctd.ctd_L0_all')
#---------------------------------------------------------------------------
# Create an input instrument
#---------------------------------------------------------------------------
instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product')
instrument_id, rev = self.rrclient.create(instrument_obj)
# Register the instrument so that the data producer and stream object are created
data_producer_id = self.damsclient.register_instrument(instrument_id)
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_input_stream_definition_to_data_process_definition(ctd_stream_def_id, dprocdef_id )
# Assert that the link between the stream definition and the data process definition was done
assocs = self.rrclient.find_associations(subject=dprocdef_id, predicate=PRED.hasInputStreamDefinition, object=ctd_stream_def_id, id_only=True)
self.assertIsNotNone(assocs)
#---------------------------------------------------------------------------
# Input Data Product
#---------------------------------------------------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
input_dp_obj = IonObject( RT.DataProduct,
name='InputDataProduct',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=ctd_stream_def_id, exchange_point='test')
#Make assertions on the input data product created
input_dp_obj = self.rrclient.read(input_dp_id)
self.assertEquals(input_dp_obj.name, 'InputDataProduct')
self.assertEquals(input_dp_obj.description, 'some new dp')
self.damsclient.assign_data_product(instrument_id, input_dp_id)
# Retrieve the stream via the DataProduct->Stream associations
stream_ids, _ = self.rrclient.find_objects(input_dp_id, PRED.hasStream, None, True)
self.in_stream_id = stream_ids[0]
#---------------------------------------------------------------------------
# Output Data Product
#---------------------------------------------------------------------------
outgoing_stream_conductivity_id = self.pubsubclient.create_stream_definition(name='conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_conductivity_id, dprocdef_id,binding='conductivity' )
outgoing_stream_pressure_id = self.pubsubclient.create_stream_definition(name='pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_pressure_id, dprocdef_id, binding='pressure' )
outgoing_stream_temperature_id = self.pubsubclient.create_stream_definition(name='temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_temperature_id, dprocdef_id, binding='temperature' )
self.output_products={}
output_dp_obj = IonObject(RT.DataProduct,
name='conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_1 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_conductivity_id)
self.output_products['conductivity'] = output_dp_id_1
output_dp_obj = IonObject(RT.DataProduct,
name='pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_2 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_pressure_id)
self.output_products['pressure'] = output_dp_id_2
output_dp_obj = IonObject(RT.DataProduct,
name='temperature',
description='transform output ',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_3 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_temperature_id)
self.output_products['temperature'] = output_dp_id_3
#---------------------------------------------------------------------------
# Create the data process
#---------------------------------------------------------------------------
def _create_data_process():
dproc_id = self.dataprocessclient.create_data_process(dprocdef_id, [input_dp_id], self.output_products)
return dproc_id
dproc_id = _create_data_process()
# Make assertions on the data process created
data_process = self.dataprocessclient.read_data_process(dproc_id)
# Assert that the data process has a process id attached
self.assertIsNotNone(data_process.process_id)
# Assert that the data process got the input data product's subscription id attached as its own input_susbcription_id attribute
self.assertIsNotNone(data_process.input_subscription_id)
output_data_product_ids = self.rrclient.find_objects(subject=dproc_id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct, id_only=True)
self.assertEquals(Set(output_data_product_ids[0]), Set([output_dp_id_1,output_dp_id_2,output_dp_id_3]))
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}})
def test_createDataProcessUsingSim(self):
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" )
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg")
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
port_agent_config = {
'device_addr': 'sbe37-simulator.oceanobservatories.org',
'device_port': 4001,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'command_port': 4002,
'data_port': 4003,
'log_level': 5,
}
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE32_CDM', parameter_dictionary_id=pdict_id)
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id)
dp_obj.name = 'ctd_raw'
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
self.output_products={}
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#-------------------------------
# Create listener for data process events and verify that events are received.
#-------------------------------
# todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events
#todo (contd) ... I believe the capability does not exist yet now. ANS And SA are not yet publishing any workflow life cycle events (Swarbhanu)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
data_process = self.rrclient.read(ctd_l0_all_data_process_id)
process_id = data_process.process_id
self.addCleanup(self.process_dispatcher.cancel_process, process_id)
#-------------------------------
# Wait until the process launched in the create_data_process() method is actually running, before proceeding further in this test
#-------------------------------
gate = ProcessStateGate(self.process_dispatcher.read_process, process_id, ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The data process (%s) did not spawn in 30 seconds" % process_id)
#-------------------------------
# Retrieve a list of all data process defintions in RR and validate that the DPD is listed
#-------------------------------
# todo: Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions
# todo: data topics are being handled by pub sub at the level of streams
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
#todo: check that activate event is received L4-CI-SA-RQ-367
#todo... (it looks like no event is being published when the data process is activated... so below, we just check for now
# todo... that the subscription is indeed activated) (Swarbhanu)
# todo: monitor process to see if it is active (sa-rq-182)
ctd_l0_all_data_process = self.rrclient.read(ctd_l0_all_data_process_id)
input_subscription_id = ctd_l0_all_data_process.input_subscription_id
subs = self.rrclient.read(input_subscription_id)
self.assertTrue(subs.activated)
# todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the extended resources for data process definition and for data process
#-------------------------------
extended_process_definition = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
self.assertEqual(1, len(extended_process_definition.data_processes))
log.debug("test_createDataProcess: extended_process_definition %s", str(extended_process_definition))
extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id)
self.assertEqual(1, len(extended_process.input_data_products))
log.debug("test_createDataProcess: extended_process %s", str(extended_process))
################################ Test the removal of data processes ##################################
#-------------------------------------------------------------------
# Test the deleting of the data process
#-------------------------------------------------------------------
# Before deleting, get the input streams, output streams and the subscriptions so that they can be checked after deleting
# dp_obj_1 = self.rrclient.read(ctd_l0_all_data_process_id)
# input_subscription_id = dp_obj_1.input_subscription_id
# out_prods, _ = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True)
# in_prods, _ = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasInputProduct, id_only=True)
# in_streams = []
# for in_prod in in_prods:
# streams, _ = self.rrclient.find_objects(in_prod, PRED.hasStream, id_only=True)
# in_streams.extend(streams)
# out_streams = []
# for out_prod in out_prods:
# streams, _ = self.rrclient.find_objects(out_prod, PRED.hasStream, id_only=True)
# out_streams.extend(streams)
# Deleting the data process
self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id)
# Check that the data process got removed. Check the lcs state. It should be retired
dp_obj = self.rrclient.read(ctd_l0_all_data_process_id)
self.assertEquals(dp_obj.lcstate, LCS.RETIRED)
# Check for process defs still attached to the data process
dpd_assn_ids = self.rrclient.find_associations(subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcessDefinition, id_only=True)
self.assertEquals(len(dpd_assn_ids), 0)
# Check for output data product still attached to the data process
out_products, assocs = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True)
self.assertEquals(len(out_products), 0)
self.assertEquals(len(assocs), 0)
# Check for input data products still attached to the data process
inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct)
self.assertEquals(len(inprod_associations), 0)
# Check for input data products still attached to the data process
inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct)
self.assertEquals(len(inprod_associations), 0)
# Check of the data process has been deactivated
self.assertIsNone(dp_obj.input_subscription_id)
# Read the original subscription id of the data process and check that it has been deactivated
with self.assertRaises(NotFound):
self.pubsubclient.read_subscription(input_subscription_id)
#-------------------------------------------------------------------
# Delete the data process definition
#-------------------------------------------------------------------
# before deleting, get the process definition being associated to in order to be able to check later if the latter gets deleted as it should
proc_def_ids, proc_def_asocs = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasProcessDefinition)
self.dataprocessclient.delete_data_process_definition(ctd_L0_all_dprocdef_id)
# check that the data process definition has been retired
dp_proc_def = self.rrclient.read(ctd_L0_all_dprocdef_id)
self.assertEquals(dp_proc_def.lcstate, LCS.RETIRED)
# Check for old associations of this data process definition
proc_defs, proc_def_asocs = self.rrclient.find_objects(ctd_L0_all_dprocdef_id, PRED.hasProcessDefinition)
self.assertEquals(len(proc_defs), 0)
# find all associations where this is the subject
_, obj_assns = self.rrclient.find_objects(subject= ctd_L0_all_dprocdef_id, id_only=True)
self.assertEquals(len(obj_assns), 0)
################################ Test the removal of data processes ##################################
# Try force delete... This should simply delete the associations and the data process object
# from the resource registry
#---------------------------------------------------------------------------------------------------------------
# Force deleting a data process
#---------------------------------------------------------------------------------------------------------------
self.dataprocessclient.force_delete_data_process(ctd_l0_all_data_process_id)
# find all associations where this is the subject
_, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True)
# find all associations where this is the object
_, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True)
self.assertEquals(len(obj_assns), 0)
self.assertEquals(len(sbj_assns), 0)
with self.assertRaises(NotFound):
self.rrclient.read(ctd_l0_all_data_process_id)
#---------------------------------------------------------------------------------------------------------------
# Force deleting a data process definition
#---------------------------------------------------------------------------------------------------------------
self.dataprocessclient.force_delete_data_process_definition(ctd_L0_all_dprocdef_id)
# find all associations where this is the subject
_, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True)
# find all associations where this is the object
_, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True)
self.assertEquals(len(obj_assns), 0)
self.assertEquals(len(sbj_assns), 0)
with self.assertRaises(NotFound):
self.rrclient.read(ctd_l0_all_data_process_id)
def test_transform_function_crd(self):
tf = TransformFunction(name='simple', module='pyon.ion.process', cls='SimpleProcess')
tf_id = self.dataprocessclient.create_transform_function(tf)
self.assertTrue(tf_id)
tf2_id = self.dataprocessclient.create_transform_function(tf)
self.assertEquals(tf_id, tf2_id)
tf_obj = self.dataprocessclient.read_transform_function(tf_id)
self.assertEquals([tf.name, tf.module, tf.cls, tf.function_type], [tf_obj.name, tf_obj.module, tf_obj.cls, tf_obj.function_type])
tf.module = 'dev.null'
self.assertRaises(BadRequest, self.dataprocessclient.create_transform_function, tf)
self.dataprocessclient.delete_transform_function(tf_id)
self.assertRaises(NotFound, self.dataprocessclient.read_transform_function, tf_id)
class TestResourceRegistry(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2coi.yml')
# Now create client to bank service
self.resource_registry_service = ResourceRegistryServiceClient(
node=self.container.node)
def test_crud(self):
# Some quick registry tests
# Can't call new with fields that aren't defined in the object's schema
with self.assertRaises(TypeError) as cm:
IonObject("UserInfo", name="name", foo="bar")
self.assertTrue(cm.exception.message ==
"__init__() got an unexpected keyword argument 'foo'")
# Can't call new with fields that aren't defined in the object's schema
with self.assertRaises(TypeError) as cm:
IonObject("UserInfo", {"name": "name", "foo": "bar"})
self.assertTrue(cm.exception.message ==
"__init__() got an unexpected keyword argument 'foo'")
# Can't call new with fields that aren't defined in the object's schema
with self.assertRaises(TypeError) as cm:
IonObject("UserInfo", {"name": "name"}, foo="bar")
self.assertTrue(cm.exception.message ==
"__init__() got an unexpected keyword argument 'foo'")
# Instantiate an object
obj = IonObject("UserInfo", name="name")
# Can set attributes that aren't in the object's schema
with self.assertRaises(AttributeError) as cm:
setattr(obj, "foo", "bar")
self.assertTrue(
cm.exception.message == "'UserInfo' object has no attribute 'foo'")
# Cam't call update with object that hasn't been persisted
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.update(obj)
self.assertTrue(
cm.exception.message.startswith(
"Object does not have required '_id' or '_rev' attribute"))
# Persist object and read it back
obj_id, obj_rev = self.resource_registry_service.create(obj)
read_obj = self.resource_registry_service.read(obj_id)
# Cannot create object with _id and _rev fields pre-set
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create(read_obj)
self.assertTrue(
cm.exception.message.startswith("Doc must not have '_id'"))
# Update object
read_obj.name = "John Doe"
self.resource_registry_service.update(read_obj)
# Update should fail with revision mismatch
with self.assertRaises(Conflict) as cm:
self.resource_registry_service.update(read_obj)
self.assertTrue(
cm.exception.message.startswith(
"Object not based on most current version"))
# Re-read and update object
read_obj = self.resource_registry_service.read(obj_id)
self.resource_registry_service.update(read_obj)
# Delete object
self.resource_registry_service.delete(obj_id)
# Make sure read, update and delete report error
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.read(obj_id)
self.assertTrue(cm.exception.message.startswith("Object with id"))
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.update(read_obj)
self.assertTrue(cm.exception.message.startswith("Object with id"))
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.delete(obj_id)
self.assertTrue(cm.exception.message.startswith("Object with id"))
# Owner creation tests
user = IonObject("ActorIdentity", name='user')
uid, _ = self.resource_registry_service.create(user)
inst = IonObject("InstrumentDevice", name='instrument')
iid, _ = self.resource_registry_service.create(
inst, headers={'ion-actor-id': str(uid)})
ids, _ = self.resource_registry_service.find_objects(iid,
PRED.hasOwner,
RT.ActorIdentity,
id_only=True)
self.assertEquals(len(ids), 1)
assoc = self.resource_registry_service.read(ids[0])
self.resource_registry_service.delete(iid)
with self.assertRaises(NotFound) as ex:
assoc = self.resource_registry_service.read(iid)
def test_lifecycle(self):
att = IonObject("InstrumentDevice", name='mine', description='desc')
rid, rev = self.resource_registry_service.create(att)
att1 = self.resource_registry_service.read(rid)
self.assertEquals(att1.name, att.name)
self.assertEquals(att1.lcstate, LCS.DRAFT_PRIVATE)
new_state = self.resource_registry_service.execute_lifecycle_transition(
rid, LCE.PLAN)
self.assertEquals(new_state, LCS.PLANNED_PRIVATE)
att2 = self.resource_registry_service.read(rid)
self.assertEquals(att2.lcstate, LCS.PLANNED_PRIVATE)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.execute_lifecycle_transition(
rid, LCE.UNANNOUNCE)
self.assertTrue(
"type=InstrumentDevice, lcstate=PLANNED_PRIVATE has no transition for event unannounce"
in cm.exception.message)
new_state = self.resource_registry_service.execute_lifecycle_transition(
rid, LCE.DEVELOP)
self.assertEquals(new_state, LCS.DEVELOPED_PRIVATE)
self.assertRaises(
iex.BadRequest,
self.resource_registry_service.execute_lifecycle_transition,
resource_id=rid,
transition_event='NONE##')
self.resource_registry_service.set_lifecycle_state(
rid, LCS.INTEGRATED_PRIVATE)
att1 = self.resource_registry_service.read(rid)
self.assertEquals(att1.lcstate, LCS.INTEGRATED_PRIVATE)
def test_association(self):
# Instantiate ActorIdentity object
actor_identity_obj = IonObject("ActorIdentity", name="name")
actor_identity_obj_id, actor_identity_obj_rev = self.resource_registry_service.create(
actor_identity_obj)
read_actor_identity_obj = self.resource_registry_service.read(
actor_identity_obj_id)
# Instantiate UserInfo object
user_info_obj = IonObject("UserInfo", name="name")
user_info_obj_id, user_info_obj_rev = self.resource_registry_service.create(
user_info_obj)
read_user_info_obj = self.resource_registry_service.read(
user_info_obj_id)
# Test create failures
with self.assertRaises(AttributeError) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.bogus, user_info_obj_id)
self.assertTrue(cm.exception.message == "bogus")
# Predicate not provided
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, None, user_info_obj_id)
self.assertTrue(
cm.exception.message == "Association must have all elements set")
# Bad association type
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id,
'bogustype')
self.assertTrue(
cm.exception.message == "Unsupported assoc_type: bogustype")
# Subject id or object not provided
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
None, PRED.hasInfo, user_info_obj_id)
self.assertTrue(
cm.exception.message == "Association must have all elements set")
# Object id or object not provided
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, None)
self.assertTrue(
cm.exception.message == "Association must have all elements set")
# Bad subject id
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.create_association(
"bogus", PRED.hasInfo, user_info_obj_id)
self.assertTrue(
cm.exception.message == "Object with id bogus does not exist.")
# Bad object id
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, "bogus")
self.assertTrue(
cm.exception.message == "Object with id bogus does not exist.")
# _id missing from subject
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
actor_identity_obj, PRED.hasInfo, user_info_obj_id)
self.assertTrue(
cm.exception.message == "Subject id or rev not available")
# _id missing from object
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, user_info_obj)
self.assertTrue(
cm.exception.message == "Object id or rev not available")
# Wrong subject type
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
user_info_obj_id, PRED.hasInfo, user_info_obj_id)
self.assertTrue(cm.exception.message ==
"Illegal subject type UserInfo for predicate hasInfo")
# Wrong object type
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, actor_identity_obj_id)
self.assertTrue(
cm.exception.message ==
"Illegal object type ActorIdentity for predicate hasInfo")
# Create two different association types between the same subject and predicate
assoc_id1, assoc_rev1 = self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
# Read object, subject
res_obj1 = self.resource_registry_service.read_object(
actor_identity_obj_id, PRED.hasInfo, RT.UserInfo)
self.assertEquals(res_obj1._id, user_info_obj_id)
res_obj1 = self.resource_registry_service.read_object(
actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, id_only=True)
self.assertEquals(res_obj1, user_info_obj_id)
res_obj2 = self.resource_registry_service.read_subject(
RT.ActorIdentity, PRED.hasInfo, user_info_obj_id)
self.assertEquals(res_obj2._id, actor_identity_obj_id)
res_obj2 = self.resource_registry_service.read_subject(
RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, id_only=True)
self.assertEquals(res_obj2, actor_identity_obj_id)
# Create a similar association to a specific revision
# TODO: This is not a supported case so far
assoc_id2, assoc_rev2 = self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, "H2R")
# Search for associations (good cases)
ret1 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
ret2 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo)
ret3 = self.resource_registry_service.find_associations(
None, PRED.hasInfo)
self.assertTrue(len(ret1) == len(ret2) == len(ret3))
self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id)
ret1 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, None, False)
ret2 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo, id_only=False)
ret3 = self.resource_registry_service.find_associations(
predicate=PRED.hasInfo, id_only=False)
self.assertTrue(ret1 == ret2 == ret3)
# Search for associations (good cases)
ret1 = self.resource_registry_service.find_associations(
read_actor_identity_obj, PRED.hasInfo, read_user_info_obj)
ret2 = self.resource_registry_service.find_associations(
read_actor_identity_obj, PRED.hasInfo)
ret3 = self.resource_registry_service.find_associations(
None, PRED.hasInfo)
self.assertTrue(len(ret1) == len(ret2) == len(ret3))
self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id)
ret1 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo, read_user_info_obj, None,
True)
ret2 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo, id_only=True)
ret3 = self.resource_registry_service.find_associations(
predicate=PRED.hasInfo, id_only=True)
self.assertTrue(ret1 == ret2 == ret3)
# Search for associations (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_associations(None, None, None)
self.assertTrue(cm.exception.message == "Illegal parameters")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_associations(
actor_identity_obj_id, None, None)
self.assertTrue(cm.exception.message == "Illegal parameters")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_associations(
None, None, user_info_obj_id)
self.assertTrue(cm.exception.message == "Illegal parameters")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_associations(
actor_identity_obj, None, user_info_obj_id)
self.assertTrue(
cm.exception.message == "Object id not available in subject")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_associations(
actor_identity_obj_id, None, user_info_obj)
self.assertTrue(
cm.exception.message == "Object id not available in object")
# Find subjects (good cases)
subj_ret1 = self.resource_registry_service.find_subjects(
RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True)
subj_ret2 = self.resource_registry_service.find_subjects(
RT.ActorIdentity, PRED.hasInfo, read_user_info_obj, True)
self.assertTrue(len(subj_ret1) == len(subj_ret2))
self.assertTrue(subj_ret1[0] == subj_ret2[0])
self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id)
subj_ret3 = self.resource_registry_service.find_subjects(
None, PRED.hasInfo, user_info_obj_id, True)
subj_ret4 = self.resource_registry_service.find_subjects(
None, None, read_user_info_obj, True)
self.assertTrue(len(subj_ret3) == len(subj_ret4))
self.assertTrue(subj_ret3[0] == subj_ret4[0])
self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id)
subj_ret5 = self.resource_registry_service.find_subjects(
None, PRED.hasInfo, user_info_obj_id, False)
subj_ret6 = self.resource_registry_service.find_subjects(
None, None, read_user_info_obj, False)
self.assertTrue(len(subj_ret5) == len(subj_ret6))
self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id)
self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id)
# Find subjects (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_subjects(None, None, None)
self.assertTrue(cm.exception.message == "Must provide object")
with self.assertRaises(AttributeError) as cm:
self.resource_registry_service.find_subjects(
RT.UserCredentials, PRED.bogus, user_info_obj_id, True)
self.assertTrue(cm.exception.message == "bogus")
ret = self.resource_registry_service.find_subjects(
RT.UserInfo, PRED.hasCredentials, user_info_obj_id, True)
self.assertTrue(len(ret[0]) == 0)
ret = self.resource_registry_service.find_subjects(
RT.UserCredentials, PRED.hasInfo, user_info_obj_id, True)
self.assertTrue(len(ret[0]) == 0)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_subjects(
RT.UserCredentials, PRED.hasInfo, user_info_obj, True)
self.assertTrue(
cm.exception.message == "Object id not available in object")
# Find objects (good cases)
subj_ret1 = self.resource_registry_service.find_objects(
actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, True)
subj_ret2 = self.resource_registry_service.find_objects(
read_actor_identity_obj, PRED.hasInfo, RT.UserInfo, True)
self.assertTrue(len(subj_ret1) == len(subj_ret2))
self.assertTrue(subj_ret1[0] == subj_ret2[0])
self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id)
subj_ret3 = self.resource_registry_service.find_objects(
actor_identity_obj_id, PRED.hasInfo, None, True)
subj_ret4 = self.resource_registry_service.find_objects(
actor_identity_obj_id, None, None, True)
self.assertTrue(len(subj_ret3) == len(subj_ret4))
self.assertTrue(subj_ret3[0] == subj_ret4[0])
self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id)
subj_ret5 = self.resource_registry_service.find_objects(
actor_identity_obj_id, PRED.hasInfo, None, False)
subj_ret6 = self.resource_registry_service.find_objects(
read_actor_identity_obj, None, None, False)
self.assertTrue(len(subj_ret5) == len(subj_ret6))
self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id)
self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id)
# Find objects (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_objects(None, None, None)
self.assertTrue(cm.exception.message == "Must provide subject")
with self.assertRaises(AttributeError) as cm:
self.resource_registry_service.find_objects(
actor_identity_obj_id, PRED.bogus, RT.UserCredentials, True)
self.assertTrue(cm.exception.message == "bogus")
ret = self.resource_registry_service.find_objects(
actor_identity_obj_id, PRED.hasCredentials, RT.ActorIdentity, True)
self.assertTrue(len(ret[0]) == 0)
ret = self.resource_registry_service.find_objects(
actor_identity_obj_id, PRED.hasInfo, RT.UserCredentials, True)
self.assertTrue(len(ret[0]) == 0)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_objects(
actor_identity_obj, PRED.hasInfo, RT.UserInfo, True)
self.assertTrue(
cm.exception.message == "Object id not available in subject")
# Get association (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.get_association(None, None, None)
self.assertTrue(cm.exception.message == "Illegal parameters")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.get_association(
actor_identity_obj_id, None, None)
self.assertTrue(cm.exception.message == "Illegal parameters")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.get_association(
None, None, user_info_obj_id)
self.assertTrue(cm.exception.message == "Illegal parameters")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.get_association(
actor_identity_obj, None, user_info_obj_id)
self.assertTrue(
cm.exception.message == "Object id not available in subject")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.get_association(
actor_identity_obj_id, None, user_info_obj)
self.assertTrue(
cm.exception.message == "Object id not available in object")
# Delete one of the associations
self.resource_registry_service.delete_association(assoc_id2)
assoc = self.resource_registry_service.get_association(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
self.assertTrue(assoc._id == assoc_id1)
# Delete (bad cases)
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.delete_association("bogus")
self.assertTrue(
cm.exception.message == "Object with id bogus does not exist.")
# Delete other association
self.resource_registry_service.delete_association(assoc_id1)
# Delete resources
self.resource_registry_service.delete(actor_identity_obj_id)
self.resource_registry_service.delete(user_info_obj_id)
def test_find_resources(self):
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_resources(
RT.UserInfo, LCS.DRAFT, "name", False)
self.assertTrue(
cm.exception.message == "find by name does not support lcstate")
ret = self.resource_registry_service.find_resources(
RT.UserInfo, None, "name", False)
self.assertTrue(len(ret[0]) == 0)
# Instantiate an object
obj = IonObject("InstrumentDevice", name="name")
# Persist object and read it back
obj_id, obj_rev = self.resource_registry_service.create(obj)
read_obj = self.resource_registry_service.read(obj_id)
ret = self.resource_registry_service.find_resources(
RT.InstrumentDevice, None, "name", False)
self.assertTrue(len(ret[0]) == 1)
self.assertTrue(ret[0][0]._id == read_obj._id)
ret = self.resource_registry_service.find_resources(
RT.InstrumentDevice, LCS.DRAFT, None, False)
self.assertTrue(len(ret[0]) == 1)
self.assertTrue(ret[0][0]._id == read_obj._id)
def test_attach(self):
binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82"
# Owner creation tests
instrument = IonObject("InstrumentDevice", name='instrument')
iid, _ = self.resource_registry_service.create(instrument)
att = Attachment(content=binary, attachment_type=AttachmentType.BLOB)
aid1 = self.resource_registry_service.create_attachment(iid, att)
att1 = self.resource_registry_service.read_attachment(aid1)
self.assertEquals(binary, att1.content)
import base64
att = Attachment(content=base64.encodestring(binary),
attachment_type=AttachmentType.ASCII)
aid2 = self.resource_registry_service.create_attachment(iid, att)
att1 = self.resource_registry_service.read_attachment(aid2)
self.assertEquals(binary, base64.decodestring(att1.content))
att_ids = self.resource_registry_service.find_attachments(iid,
id_only=True)
self.assertEquals(att_ids, [aid1, aid2])
att_ids = self.resource_registry_service.find_attachments(
iid, id_only=True, descending=True)
self.assertEquals(att_ids, [aid2, aid1])
att_ids = self.resource_registry_service.find_attachments(
iid, id_only=True, descending=True, limit=1)
self.assertEquals(att_ids, [aid2])
atts = self.resource_registry_service.find_attachments(iid,
id_only=False,
limit=1)
self.assertEquals(atts[0].content, att1.content)
self.resource_registry_service.delete_attachment(aid1)
att_ids = self.resource_registry_service.find_attachments(iid,
id_only=True)
self.assertEquals(att_ids, [aid2])
self.resource_registry_service.delete_attachment(aid2)
att_ids = self.resource_registry_service.find_attachments(iid,
id_only=True)
self.assertEquals(att_ids, [])
def test_read_mult(self):
test_resource1_id, _ = self.resource_registry_service.create(
Resource(name='test1'))
test_resource2_id, _ = self.resource_registry_service.create(
Resource(name='test2'))
res_list = [test_resource1_id, test_resource2_id]
objects = self.resource_registry_service.read_mult(res_list)
for o in objects:
self.assertIsInstance(o, Resource)
self.assertTrue(o._id in res_list)
def test_find_associations_mult(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.resource_registry_service.create(dp)
transform_id, _ = self.resource_registry_service.create(transform)
pd_id, _ = self.resource_registry_service.create(pd)
self.resource_registry_service.create_association(
subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.resource_registry_service.create_association(
subject=transform_id,
object=pd_id,
predicate=PRED.hasProcessDefinition)
results, _ = self.resource_registry_service.find_associations_mult(
subjects=[dp_id], id_only=True)
self.assertTrue(results == [transform_id])
results, _ = self.resource_registry_service.find_associations_mult(
subjects=[dp_id, transform_id], id_only=True)
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.identcli = IdentityManagementServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
self.stream_def_id = self.pubsubcli.create_stream_definition(
name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(
name='ingestion worker')
ingestion_worker_definition.executable = {
'module':
'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class': 'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(
process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(
self.process_definitions['ingestion_worker'], configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
@attr('EXT')
@attr('PREP')
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data',
parameter_dictionary_id=parameter_dictionary._id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
# Assert that the data product has an associated stream at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id,
PRED.hasStreamDefinition,
RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp')
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream,
RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition,
RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
group_names = self.dpsc_cli.get_data_product_group_list()
self.assertIn("PRODNAME", group_names)
#----------------------------------------------------------------------------------------
# Create users then notifications to this data product for each user
#----------------------------------------------------------------------------------------
# user_1
user_1 = UserInfo()
user_1.name = 'user_1'
user_1.contact.email = '*****@*****.**'
# user_2
user_2 = UserInfo()
user_2.name = 'user_2'
user_2.contact.email = '*****@*****.**'
#user1 is a complete user
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
actor_id = self.identcli.create_actor_identity(actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials",
{"name": self.subject})
self.identcli.register_user_credentials(actor_id, user_credentials_obj)
user_id_1 = self.identcli.create_user_info(actor_id, user_1)
user_id_2, _ = self.rrclient.create(user_2)
delivery_config1a = IonObject(
OT.DeliveryConfiguration,
email='*****@*****.**',
mode=DeliveryModeEnum.EMAIL,
frequency=NotificationFrequencyEnum.BATCH)
delivery_config1b = IonObject(
OT.DeliveryConfiguration,
email='*****@*****.**',
mode=DeliveryModeEnum.EMAIL,
frequency=NotificationFrequencyEnum.BATCH)
notification_request_1 = NotificationRequest(
name="notification_1",
origin=dp_id,
origin_type="type_1",
event_type=OT.ResourceLifecycleEvent,
disabled_by_system=False,
delivery_configurations=[delivery_config1a, delivery_config1b])
delivery_config2a = IonObject(
OT.DeliveryConfiguration,
email='*****@*****.**',
mode=DeliveryModeEnum.EMAIL,
frequency=NotificationFrequencyEnum.BATCH)
delivery_config2b = IonObject(
OT.DeliveryConfiguration,
email='*****@*****.**',
mode=DeliveryModeEnum.EMAIL,
frequency=NotificationFrequencyEnum.BATCH)
notification_request_2 = NotificationRequest(
name="notification_2",
origin=dp_id,
origin_type="type_2",
disabled_by_system=False,
event_type=OT.DetectionEvent,
delivery_configurations=[delivery_config2a, delivery_config2b])
notification_request_1_id = self.unsc.create_notification(
notification=notification_request_1, user_id=user_id_1)
notification_request_2_id = self.unsc.create_notification(
notification=notification_request_2, user_id=user_id_2)
self.unsc.delete_notification(notification_request_1_id)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description, 'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
#validate that there is one active and one retired user notification for this data product
self.assertEqual(
1, len(extended_product.computed.active_user_subscriptions.value))
self.assertEqual(
1, len(extended_product.computed.past_user_subscriptions.value))
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(
ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(
0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
def ion_object_encoder(obj):
return obj.__dict__
#test prepare for create
data_product_data = self.dpsc_cli.prepare_data_product_support()
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, "")
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].resources),
2)
self.assertEqual(
len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].
associated_resources), 0)
self.assertEqual(
len(data_product_data.associations['Dataset'].associated_resources
), 0)
#test prepare for update
data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, dp_id)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].resources),
2)
self.assertEqual(
len(data_product_data.associations['Dataset'].resources), 1)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].
associated_resources), 1)
self.assertEqual(
data_product_data.associations['StreamDefinition'].
associated_resources[0].s, dp_id)
self.assertEqual(
len(data_product_data.associations['Dataset'].associated_resources
), 1)
self.assertEqual(
data_product_data.associations['Dataset'].associated_resources[0].
s, dp_id)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
# Assert that there are no associated streams leftover after deleting the data product
stream_ids, assoc_ids = self.rrclient.find_objects(
dp_id, PRED.hasStream, RT.Stream, True)
self.assertEquals(len(stream_ids), 0)
self.assertEquals(len(assoc_ids), 0)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data', parameter_dictionary_id=pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_derived_data_product(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition,
ctd_stream_def_id)
dp = DataProduct(name='Instrument DP')
dp_id = self.dpsc_cli.create_data_product(
dp, stream_definition_id=ctd_stream_def_id)
self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id,
predicate=PRED.hasDataset,
id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" %
str(dp_id))
dataset_id = dataset_ids[0]
# Make the derived data product
simple_stream_def_id = self.pubsubcli.create_stream_definition(
name='TEMPWAT stream def',
parameter_dictionary_id=pdict_id,
available_fields=['time', 'temp'])
tempwat_dp = DataProduct(name='TEMPWAT',
category=DataProductTypeEnum.DERIVED)
tempwat_dp_id = self.dpsc_cli.create_data_product(
tempwat_dp,
stream_definition_id=simple_stream_def_id,
parent_data_product_id=dp_id)
self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id)
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
rdt['pressure'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb,
origin=dataset_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id,
PRED.hasDataset,
id_only=True)
tempwat_dataset_id = tempwat_dataset_ids[0]
granule = self.data_retriever.retrieve(
tempwat_dataset_id, delivery_format=simple_stream_def_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
self.assertEquals(set(rdt.fields), set(['time', 'temp']))
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# Subscribe to persist events
#------------------------------------------------------------------------------------------------
queue = gevent.queue.Queue()
def info_event_received(message, headers):
queue.put(message)
es = EventSubscriber(event_type=OT.InformationContentStatusEvent,
callback=info_event_received,
origin=dp_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id,
predicate=PRED.hasDataset,
id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" %
str(dp_id))
dataset_id = dataset_ids[0]
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb,
origin=dataset_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug(
"The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'"
)
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name, 'DP1')
self.assertEquals(data_product_object.description, 'some new dp')
log.debug(
"Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" %
(dp_obj.name, dp_obj.description, data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
dataset_modified.clear()
rdt['time'] = np.arange(20, 40)
publisher.publish(rdt.to_granule())
self.assertFalse(dataset_modified.wait(2))
self.dpsc_cli.activate_data_product_persistence(dp_id)
dataset_modified.clear()
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt['time'], np.arange(40))
dataset_ids, _ = self.rrclient.find_objects(dp_id,
PRED.hasDataset,
id_only=True)
self.assertEquals(len(dataset_ids), 1)
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
info_event_counter = 0
runtime = 0
starttime = time.time()
caught_events = []
#check that the four InfoStatusEvents were received
while info_event_counter < 4 and runtime < 60:
a = queue.get(timeout=60)
caught_events.append(a)
info_event_counter += 1
runtime = time.time() - starttime
self.assertEquals(info_event_counter, 4)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
datastore_name = CACHE_DATASTORE_NAME
self.db = self.container.datastore_manager.get_datastore(
datastore_name)
self.stream_def_id = self.pubsubcli.create_stream_definition(
name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(
name='ingestion worker')
ingestion_worker_definition.executable = {
'module':
'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class': 'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(
process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(
self.process_definitions['ingestion_worker'], configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
@attr('EXT')
@attr('PREP')
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data',
parameter_dictionary_id=parameter_dictionary._id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Generic time-series data domain creation
tdom, sdom = time_series_domain()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain=tdom.dump(),
spatial_domain=sdom.dump())
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
# Assert that the data product has an associated stream at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id,
PRED.hasStreamDefinition,
RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp',
temporal_domain=tdom.dump(),
spatial_domain=sdom.dump())
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream,
RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition,
RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
group_names = self.dpsc_cli.get_data_product_group_list()
self.assertIn("PRODNAME", group_names)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description, 'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(
ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(
0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
def ion_object_encoder(obj):
return obj.__dict__
#test prepare for create
data_product_data = self.dpsc_cli.prepare_data_product_support()
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, "")
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].resources),
2)
self.assertEqual(
len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].
associated_resources), 0)
self.assertEqual(
len(data_product_data.associations['Dataset'].associated_resources
), 0)
#test prepare for update
data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, dp_id)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].resources),
2)
self.assertEqual(
len(data_product_data.associations['Dataset'].resources), 1)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].
associated_resources), 1)
self.assertEqual(
data_product_data.associations['StreamDefinition'].
associated_resources[0].s, dp_id)
self.assertEqual(
len(data_product_data.associations['Dataset'].associated_resources
), 1)
self.assertEqual(
data_product_data.associations['Dataset'].associated_resources[0].
s, dp_id)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
# Assert that there are no associated streams leftover after deleting the data product
stream_ids, assoc_ids = self.rrclient.find_objects(
dp_id, PRED.hasStream, RT.Stream, True)
self.assertEquals(len(stream_ids), 0)
self.assertEquals(len(assoc_ids), 0)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data', parameter_dictionary_id=pdict_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain=tdom,
spatial_domain=sdom)
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_derived_data_product(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition,
ctd_stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='Instrument DP',
temporal_domain=tdom.dump(),
spatial_domain=sdom.dump())
dp_id = self.dpsc_cli.create_data_product(
dp, stream_definition_id=ctd_stream_def_id)
self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id,
predicate=PRED.hasDataset,
id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" %
str(dp_id))
dataset_id = dataset_ids[0]
# Make the derived data product
simple_stream_def_id = self.pubsubcli.create_stream_definition(
name='TEMPWAT stream def',
parameter_dictionary_id=pdict_id,
available_fields=['time', 'temp'])
tempwat_dp = DataProduct(name='TEMPWAT')
tempwat_dp_id = self.dpsc_cli.create_data_product(
tempwat_dp,
stream_definition_id=simple_stream_def_id,
parent_data_product_id=dp_id)
self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id)
self.dpsc_cli.activate_data_product_persistence(tempwat_dp_id)
self.addCleanup(self.dpsc_cli.suspend_data_product_persistence,
tempwat_dp_id)
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
rdt['pressure'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb,
origin=dataset_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id,
PRED.hasDataset,
id_only=True)
tempwat_dataset_id = tempwat_dataset_ids[0]
granule = self.data_retriever.retrieve(
tempwat_dataset_id, delivery_format=simple_stream_def_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
self.assertEquals(set(rdt.fields), set(['time', 'temp']))
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain=tdom,
spatial_domain=sdom)
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id,
predicate=PRED.hasDataset,
id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" %
str(dp_id))
dataset_id = dataset_ids[0]
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb,
origin=dataset_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug(
"The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'"
)
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name, 'DP1')
self.assertEquals(data_product_object.description, 'some new dp')
log.debug(
"Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" %
(dp_obj.name, dp_obj.description, data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
dataset_modified.clear()
rdt['time'] = np.arange(20, 40)
publisher.publish(rdt.to_granule())
self.assertFalse(dataset_modified.wait(2))
self.dpsc_cli.activate_data_product_persistence(dp_id)
dataset_modified.clear()
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt['time'], np.arange(40))
dataset_ids, _ = self.rrclient.find_objects(dp_id,
PRED.hasDataset,
id_only=True)
self.assertEquals(len(dataset_ids), 1)
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
def test_lookup_values(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsubcli.create_stream_definition(
'lookup', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, stream_def_id)
data_product = DataProduct(name='lookup data product')
tdom, sdom = time_series_domain()
data_product.temporal_domain = tdom.dump()
data_product.spatial_domain = sdom.dump()
data_product_id = self.dpsc_cli.create_data_product(
data_product, stream_definition_id=stream_def_id)
self.addCleanup(self.dpsc_cli.delete_data_product, data_product_id)
data_producer = DataProducer(name='producer')
data_producer.producer_context = DataProcessProducerContext()
data_producer.producer_context.configuration['qc_keys'] = [
'offset_document'
]
data_producer_id, _ = self.rrclient.create(data_producer)
self.addCleanup(self.rrclient.delete, data_producer_id)
assoc, _ = self.rrclient.create_association(
subject=data_product_id,
object=data_producer_id,
predicate=PRED.hasDataProducer)
self.addCleanup(self.rrclient.delete_association, assoc)
document_keys = self.damsclient.list_qc_references(data_product_id)
self.assertEquals(document_keys, ['offset_document'])
svm = StoredValueManager(self.container)
svm.stored_value_cas('offset_document', {'offset_a': 2.0})
self.dpsc_cli.activate_data_product_persistence(data_product_id)
dataset_ids, _ = self.rrclient.find_objects(subject=data_product_id,
predicate=PRED.hasDataset,
id_only=True)
dataset_id = dataset_ids[0]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temp'] = [20.]
granule = rdt.to_granule()
stream_ids, _ = self.rrclient.find_objects(subject=data_product_id,
predicate=PRED.hasStream,
id_only=True)
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['temp'], rdt2['temp'])
np.testing.assert_array_almost_equal(rdt2['calibrated'],
np.array([22.0]))
svm.stored_value_cas('updated_document', {'offset_a': 3.0})
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ep = EventPublisher(event_type=OT.ExternalReferencesUpdatedEvent)
ep.publish_event(origin=data_product_id,
reference_keys=['updated_document'])
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [1]
rdt['temp'] = [20.]
granule = rdt.to_granule()
gevent.sleep(2) # Yield so that the event goes through
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt2['temp'], np.array([20., 20.]))
np.testing.assert_array_almost_equal(rdt2['calibrated'],
np.array([22.0, 23.0]))
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
sensor_model_id #is only a target
#create a parsed product for this instrument output
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain = tdom,
spatial_domain = sdom)
pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id)
# Lifecycle
self.assertEquals(len(extended_instrument.lcstate_transitions), 7)
self.assertEquals(set(extended_instrument.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce']))
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
log.debug("extended_instrument.computed: %s", extended_instrument.computed)
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name)
#check agent instance
inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id)
self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
#compound assoc return list of lists so check the first element
self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name)
extended_platform = self.IMS.get_platform_device_extension(platform_device_id)
self.assertEqual(1, len(extended_platform.instrument_devices))
self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id)
self.assertEqual(1, len(extended_platform.instrument_models))
self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id)
self.assertEquals(extended_platform.platform_agent._id, platform_agent_id)
self.assertEquals(len(extended_platform.lcstate_transitions), 7)
self.assertEquals(set(extended_platform.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce']))
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
#check data_product_parameters_set
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.data_product_parameters_set.status)
self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value)
# the ctd parameters should include 'temp'
self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical'])
#none of these will work because there is no agent
self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
extended_instrument.computed.firmware_version.status)
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.operational_state.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.power_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.communications_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.data_status_roll_up.status)
# self.assertEqual(StatusType.STATUS_OK,
# extended_instrument.computed.data_status_roll_up.value)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.location_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.recent_events.status)
# self.assertEqual([], extended_instrument.computed.recent_events.value)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.RR2.pluck(sensor_device_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
def test_custom_attributes(self):
"""
Test assignment of custom attributes
"""
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel,
{"custom_attributes":
{"favorite_color": "attr desc goes here"}
}))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice,
{"custom_attributes":
{"favorite_color": "red",
"bogus_attr": "should raise warning"
}
}))
self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
# cleanup
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
def _get_datastore(self, dataset_id):
dataset = self.DSC.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_resource_state_save_restore(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg",
stream_configurations = [raw_config, parsed_config] )
instAgent_id = self.IMS.create_instrument_agent(instAgent_obj)
log.debug( 'new InstrumentAgent id = %s', instAgent_id)
self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 '
+ '(SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
log.debug( 'new dp_id = %s', str(data_product_id2))
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
# spin up agent
self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.IMS.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
instance_obj.agent_process_id)
# take snapshot of config
snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot")
snap_obj = self.RR.read_attachment(snap_id, include_content=True)
print "Saved config:"
print snap_obj.content
#modify config
instance_obj.driver_config["comms_config"] = "BAD_DATA"
self.RR.update(instance_obj)
#restore config
self.IMS.restore_resource_state(instDevice_id, snap_id)
instance_obj = self.RR.read(instAgentInstance_id)
self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"])
self.DP.delete_data_product(data_product_id1)
self.DP.delete_data_product(data_product_id2)
def test_agent_instance_config(self):
"""
Verify that agent configurations are being built properly
"""
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_obj = any_old(RT.Org)
org_id = self.RR2.create(org_obj)
inst_startup_config = {'startup': 'config'}
generic_alerts_config = {'lvl1': {'lvl2': 'lvl3val'}}
required_config_keys = [
'org_name',
'device_type',
'agent',
'driver_config',
'stream_config',
'startup_config',
'aparam_alert_config',
'children']
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('aparam_alert_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alert_config'])
self.assertIn('stream_config', config)
for key in ['children']:
self.assertEqual({}, config[key])
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('aparam_alert_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alert_config'])
self.assertIn('stream_config', config)
self.assertIn('driver_config', config)
self.assertIn('foo', config['driver_config'])
self.assertEqual('bar', config['driver_config']['foo'])
self.assertIn('process_type', config['driver_config'])
self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type'])
if None is inst_device_id:
for key in ['children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id], inst_device_id)
def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertIn('process_type', config['driver_config'])
self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('aparam_alert_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alert_config'])
self.assertIn('stream_config', config)
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': {'foo': 'bar'},
'alerts': generic_alerts_config})
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config,
'alerts': generic_alerts_config})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5 )
instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)
log.debug("Making the structure for a platform agent, which will be the child")
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure()
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj)
child_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug("Making the structure for a platform agent, which will be the parent")
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure()
platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure()
instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
full_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)
class TestObservatoryManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.org_management_service = OrgManagementServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#print 'TestObservatoryManagementServiceIntegration: started services'
self.event_publisher = EventPublisher()
# @unittest.skip('this exists only for debugging the launch process')
# def test_just_the_setup(self):
# return
def destroy(self, resource_ids):
self.OMS.force_delete_observatory(resource_ids.observatory_id)
self.OMS.force_delete_subsite(resource_ids.subsite_id)
self.OMS.force_delete_subsite(resource_ids.subsite2_id)
self.OMS.force_delete_subsite(resource_ids.subsiteb_id)
self.OMS.force_delete_subsite(resource_ids.subsitez_id)
self.OMS.force_delete_platform_site(resource_ids.platform_site_id)
self.OMS.force_delete_platform_site(resource_ids.platform_siteb_id)
self.OMS.force_delete_platform_site(resource_ids.platform_siteb2_id)
self.OMS.force_delete_platform_site(resource_ids.platform_site3_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_site_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_site2_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_siteb3_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_site4_id)
#@unittest.skip('targeting')
def test_resources_associations(self):
resources = self._make_associations()
self.destroy(resources)
#@unittest.skip('targeting')
def test_find_related_frames_of_reference(self):
# finding subordinates gives a dict of obj lists, convert objs to ids
def idify(adict):
ids = {}
for k, v in adict.iteritems():
ids[k] = []
for obj in v:
ids[k].append(obj._id)
return ids
# a short version of the function we're testing, with id-ify
def short(resource_id, output_types):
ret = self.OMS.find_related_frames_of_reference(resource_id,
output_types)
return idify(ret)
#set up associations first
stuff = self._make_associations()
#basic traversal of tree from instrument to platform
ids = short(stuff.instrument_site_id, [RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
#since this is the first search, just make sure the input inst_id got stripped
if RT.InstrumentSite in ids:
self.assertNotIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
#basic traversal of tree from platform to instrument
ids = short(stuff.platform_siteb_id, [RT.InstrumentSite])
self.assertIn(RT.InstrumentSite, ids)
self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
self.assertNotIn(stuff.instrument_site2_id, ids[RT.InstrumentSite])
#full traversal of tree from observatory down to instrument
ids = short(stuff.observatory_id, [RT.InstrumentSite])
self.assertIn(RT.InstrumentSite, ids)
self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
#full traversal of tree from instrument to observatory
ids = short(stuff.instrument_site_id, [RT.Observatory])
self.assertIn(RT.Observatory, ids)
self.assertIn(stuff.observatory_id, ids[RT.Observatory])
#partial traversal, only down to platform
ids = short(stuff.observatory_id, [RT.Subsite, RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(RT.Subsite, ids)
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_site3_id, ids[RT.PlatformSite])
self.assertIn(stuff.subsite_id, ids[RT.Subsite])
self.assertIn(stuff.subsite2_id, ids[RT.Subsite])
self.assertIn(stuff.subsitez_id, ids[RT.Subsite])
self.assertIn(stuff.subsiteb_id, ids[RT.Subsite])
self.assertNotIn(RT.InstrumentSite, ids)
#partial traversal, only down to platform
ids = short(stuff.instrument_site_id, [RT.Subsite, RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(RT.Subsite, ids)
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.subsite_id, ids[RT.Subsite])
self.assertIn(stuff.subsiteb_id, ids[RT.Subsite])
self.assertNotIn(stuff.subsite2_id, ids[RT.Subsite])
self.assertNotIn(stuff.subsitez_id, ids[RT.Subsite])
self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
self.assertNotIn(RT.Observatory, ids)
self.destroy(stuff)
def _make_associations(self):
"""
create one of each resource and association used by OMS
to guard against problems in ion-definitions
"""
#raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41")
"""
the tree we're creating (observatory, sites, platforms, instruments)
rows are lettered, colums numbered.
- first row is implied a
- first column is implied 1
- site Z, just because
O--Sz
|
S--S2--P3--I4
|
Sb-Pb2-Ib3
|
P--I2 <- PlatformDevice, InstrumentDevice2
|
Pb <- PlatformDevice b
|
I <- InstrumentDevice
"""
org_id = self.OMS.create_marine_facility(any_old(RT.Org))
def create_under_org(resource_type):
obj = any_old(resource_type)
if RT.InstrumentDevice == resource_type:
resource_id = self.IMS.create_instrument_device(obj)
else:
resource_id, _ = self.RR.create(obj)
self.OMS.assign_resource_to_observatory_org(resource_id=resource_id, org_id=org_id)
return resource_id
#stuff we control
observatory_id = create_under_org(RT.Observatory)
subsite_id = create_under_org(RT.Subsite)
subsite2_id = create_under_org(RT.Subsite)
subsiteb_id = create_under_org(RT.Subsite)
subsitez_id = create_under_org(RT.Subsite)
platform_site_id = create_under_org(RT.PlatformSite)
platform_siteb_id = create_under_org(RT.PlatformSite)
platform_siteb2_id = create_under_org(RT.PlatformSite)
platform_site3_id = create_under_org(RT.PlatformSite)
instrument_site_id = create_under_org(RT.InstrumentSite)
instrument_site2_id = create_under_org(RT.InstrumentSite)
instrument_siteb3_id = create_under_org(RT.InstrumentSite)
instrument_site4_id = create_under_org(RT.InstrumentSite)
#stuff we associate to
instrument_device_id = create_under_org(RT.InstrumentDevice)
instrument_device2_id = create_under_org(RT.InstrumentDevice)
platform_device_id = create_under_org(RT.PlatformDevice)
platform_deviceb_id = create_under_org(RT.PlatformDevice)
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
deployment_id, _ = self.RR.create(any_old(RT.Deployment))
#observatory
self.RR.create_association(observatory_id, PRED.hasSite, subsite_id)
self.RR.create_association(observatory_id, PRED.hasSite, subsitez_id)
#site
self.RR.create_association(subsite_id, PRED.hasSite, subsite2_id)
self.RR.create_association(subsite_id, PRED.hasSite, subsiteb_id)
self.RR.create_association(subsite2_id, PRED.hasSite, platform_site3_id)
self.RR.create_association(subsiteb_id, PRED.hasSite, platform_siteb2_id)
self.RR.create_association(subsiteb_id, PRED.hasSite, platform_site_id)
#platform_site(s)
self.RR.create_association(platform_site3_id, PRED.hasSite, instrument_site4_id)
self.RR.create_association(platform_siteb2_id, PRED.hasSite, instrument_siteb3_id)
self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site2_id)
self.RR.create_association(platform_site_id, PRED.hasSite, platform_siteb_id)
self.RR.create_association(platform_siteb_id, PRED.hasSite, instrument_site_id)
self.RR.create_association(platform_siteb_id, PRED.hasDevice, platform_deviceb_id)
self.RR.create_association(platform_site_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id)
self.RR.create_association(platform_site_id, PRED.hasDeployment, deployment_id)
#instrument_site(s)
self.RR.create_association(instrument_site_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id)
self.RR.create_association(instrument_site_id, PRED.hasDeployment, deployment_id)
self.RR.create_association(instrument_site2_id, PRED.hasDevice, instrument_device2_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id)
ret = DotDict()
ret.org_id = org_id
ret.observatory_id = observatory_id
ret.subsite_id = subsite_id
ret.subsite2_id = subsite2_id
ret.subsiteb_id = subsiteb_id
ret.subsitez_id = subsitez_id
ret.platform_site_id = platform_site_id
ret.platform_siteb_id = platform_siteb_id
ret.platform_siteb2_id = platform_siteb2_id
ret.platform_site3_id = platform_site3_id
ret.instrument_site_id = instrument_site_id
ret.instrument_site2_id = instrument_site2_id
ret.instrument_siteb3_id = instrument_siteb3_id
ret.instrument_site4_id = instrument_site4_id
ret.instrument_device_id = instrument_device_id
ret.instrument_device2_id = instrument_device2_id
ret.platform_device_id = platform_device_id
ret.platform_deviceb_id = platform_deviceb_id
ret.instrument_model_id = instrument_model_id
ret.platform_model_id = platform_model_id
ret.deployment_id = deployment_id
return ret
#@unittest.skip("targeting")
def test_create_observatory(self):
observatory_obj = IonObject(RT.Observatory,
name='TestFacility',
description='some new mf')
observatory_id = self.OMS.create_observatory(observatory_obj)
self.OMS.force_delete_observatory(observatory_id)
#@unittest.skip("targeting")
def test_find_observatory_org(self):
org_obj = IonObject(RT.Org,
name='TestOrg',
description='some new mf org')
org_id = self.OMS.create_marine_facility(org_obj)
observatory_obj = IonObject(RT.Observatory,
name='TestObservatory',
description='some new obs')
observatory_id = self.OMS.create_observatory(observatory_obj)
#make association
self.OMS.assign_resource_to_observatory_org(observatory_id, org_id)
#find association
org_objs = self.OMS.find_org_by_observatory(observatory_id)
self.assertEqual(1, len(org_objs))
self.assertEqual(org_id, org_objs[0]._id)
print("org_id=<" + org_id + ">")
#create a subsite with parent Observatory
subsite_obj = IonObject(RT.Subsite,
name= 'TestSubsite',
description = 'sample subsite')
subsite_id = self.OMS.create_subsite(subsite_obj, observatory_id)
self.assertIsNotNone(subsite_id, "Subsite not created.")
# verify that Subsite is linked to Observatory
mf_subsite_assoc = self.RR.get_association(observatory_id, PRED.hasSite, subsite_id)
self.assertIsNotNone(mf_subsite_assoc, "Subsite not connected to Observatory.")
# add the Subsite as a resource of this Observatory
self.OMS.assign_resource_to_observatory_org(resource_id=subsite_id, org_id=org_id)
# verify that Subsite is linked to Org
org_subsite_assoc = self.RR.get_association(org_id, PRED.hasResource, subsite_id)
self.assertIsNotNone(org_subsite_assoc, "Subsite not connected as resource to Org.")
#create a logical platform with parent Subsite
platform_site_obj = IonObject(RT.PlatformSite,
name= 'TestPlatformSite',
description = 'sample logical platform')
platform_site_id = self.OMS.create_platform_site(platform_site_obj, subsite_id)
self.assertIsNotNone(platform_site_id, "PlatformSite not created.")
# verify that PlatformSite is linked to Site
site_lp_assoc = self.RR.get_association(subsite_id, PRED.hasSite, platform_site_id)
self.assertIsNotNone(site_lp_assoc, "PlatformSite not connected to Site.")
# add the PlatformSite as a resource of this Observatory
self.OMS.assign_resource_to_observatory_org(resource_id=platform_site_id, org_id=org_id)
# verify that PlatformSite is linked to Org
org_lp_assoc = self.RR.get_association(org_id, PRED.hasResource, platform_site_id)
self.assertIsNotNone(org_lp_assoc, "PlatformSite not connected as resource to Org.")
#create a logical instrument with parent logical platform
instrument_site_obj = IonObject(RT.InstrumentSite,
name= 'TestInstrumentSite',
description = 'sample logical instrument')
instrument_site_id = self.OMS.create_instrument_site(instrument_site_obj, platform_site_id)
self.assertIsNotNone(instrument_site_id, "InstrumentSite not created.")
# verify that InstrumentSite is linked to PlatformSite
li_lp_assoc = self.RR.get_association(platform_site_id, PRED.hasSite, instrument_site_id)
self.assertIsNotNone(li_lp_assoc, "InstrumentSite not connected to PlatformSite.")
# add the InstrumentSite as a resource of this Observatory
self.OMS.assign_resource_to_observatory_org(resource_id=instrument_site_id, org_id=org_id)
# verify that InstrumentSite is linked to Org
org_li_assoc = self.RR.get_association(org_id, PRED.hasResource, instrument_site_id)
self.assertIsNotNone(org_li_assoc, "InstrumentSite not connected as resource to Org.")
# remove the InstrumentSite as a resource of this Observatory
self.OMS.unassign_resource_from_observatory_org(instrument_site_id, org_id)
# verify that InstrumentSite is linked to Org
assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.InstrumentSite, id_only=True )
self.assertEqual(len(assocs), 0)
# remove the InstrumentSite
self.OMS.delete_instrument_site(instrument_site_id)
assocs, _ = self.RR.find_objects(platform_site_id, PRED.hasSite, RT.InstrumentSite, id_only=True )
self.assertEqual(len(assocs), 1)
#todo: remove the dangling association
# remove the PlatformSite as a resource of this Observatory
self.OMS.unassign_resource_from_observatory_org(platform_site_id, org_id)
# verify that PlatformSite is linked to Org
assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.PlatformSite, id_only=True )
self.assertEqual(len(assocs), 0)
# remove the Site as a resource of this Observatory
self.OMS.unassign_resource_from_observatory_org(subsite_id, org_id)
# verify that Site is linked to Org
assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.Subsite, id_only=True )
self.assertEqual(len(assocs), 0)
self.RR.delete(org_id)
self.OMS.force_delete_observatory(observatory_id)
self.OMS.force_delete_subsite(subsite_id)
self.OMS.force_delete_platform_site(platform_site_id)
self.OMS.force_delete_instrument_site(instrument_site_id)
#@unittest.skip("in development...")
@attr('EXT')
def test_observatory_org_extended(self):
stuff = self._make_associations()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',
id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed',
parameter_dictionary_id=parsed_pdict_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=stuff.instrument_device_id,
data_product_id=data_product_id1)
#--------------------------------------------------------------------------------
# Get the extended Site (platformSite)
#--------------------------------------------------------------------------------
extended_site = self.OMS.get_site_extension(stuff.platform_site_id)
log.debug("extended_site: %s ", str(extended_site))
self.assertEqual(1, len(extended_site.platform_devices))
self.assertEqual(1, len(extended_site.platform_models))
self.assertEqual(stuff.platform_device_id, extended_site.platform_devices[0]._id)
self.assertEqual(stuff.platform_model_id, extended_site.platform_models[0]._id)
#--------------------------------------------------------------------------------
# Get the extended Org
#--------------------------------------------------------------------------------
#test the extended resource
extended_org = self.org_management_service.get_marine_facility_extension(stuff.org_id)
log.debug("test_observatory_org_extended: extended_org: %s ", str(extended_org))
#self.assertEqual(2, len(extended_org.instruments_deployed) )
#self.assertEqual(1, len(extended_org.platforms_not_deployed) )
self.assertEqual(2, extended_org.number_of_platforms)
self.assertEqual(2, len(extended_org.platform_models) )
self.assertEqual(2, extended_org.number_of_instruments)
self.assertEqual(2, len(extended_org.instrument_models) )
#test the extended resource of the ION org
ion_org_id = self.org_management_service.find_org()
extended_org = self.org_management_service.get_marine_facility_extension(ion_org_id._id, user_id=12345)
log.debug("test_observatory_org_extended: extended_ION_org: %s ", str(extended_org))
self.assertEqual(0, len(extended_org.members))
self.assertEqual(0, extended_org.number_of_platforms)
#self.assertEqual(1, len(extended_org.sites))
#--------------------------------------------------------------------------------
# Get the extended Site
#--------------------------------------------------------------------------------
#create device state events to use for op /non-op filtering in extended
t = get_ion_ts()
self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent',
origin = stuff.instrument_device_id, state=ResourceAgentState.STREAMING )
self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent',
origin = stuff.instrument_device2_id, state=ResourceAgentState.INACTIVE )
extended_site = self.OMS.get_site_extension(stuff.instrument_site2_id)
log.debug("test_observatory_org_extended: extended_site: %s ", str(extended_site))
self.dpclient.delete_data_product(data_product_id1)
class UserNotificationIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.unsc = UserNotificationServiceClient(node=self.container.node)
self.rrc = ResourceRegistryServiceClient(node=self.container.node)
self.imc = IdentityManagementServiceClient(node=self.container.node)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_email(self):
proc1 = self.container.proc_manager.procs_by_name['user_notification']
# Create a user and get the user_id
user = UserInfo(name='new_user')
user_id, _ = self.rrc.create(user)
# set up....
notification_id = self.unsc.create_email(
event_type='ResourceLifecycleEvent',
event_subtype=None,
origin='Some_Resource_Agent_ID1',
origin_type=None,
user_id=user_id,
email='*****@*****.**',
mode=DeliveryMode.DIGEST,
message_header='message_header',
parser='parser',
period=1)
#------------------------------------------------------------------------------------------------------
# Setup so as to be able to get the message and headers going into the
# subscription callback method of the EmailEventProcessor
#------------------------------------------------------------------------------------------------------
# publish an event for each notification to generate the emails
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event")
msg_tuple = proc1.event_processors[
notification_id].smtp_client.sentmail.get(timeout=4)
self.assertTrue(proc1.event_processors[notification_id].smtp_client.
sentmail.empty())
message = msg_tuple[2]
list_lines = message.split("\n")
#-------------------------------------------------------
# parse the message body
#-------------------------------------------------------
message_dict = {}
for line in list_lines:
key_item = line.split(": ")
if key_item[0] == 'Subject':
message_dict['Subject'] = key_item[1] + key_item[2]
else:
try:
message_dict[key_item[0]] = key_item[1]
except IndexError as exc:
# these IndexError exceptions happen only because the message sometimes
# has successive /r/n (i.e. new lines) and therefore,
# the indexing goes out of range. These new lines
# can just be ignored. So we ignore the exceptions here.
pass
#-------------------------------------------------------
# make assertions
#-------------------------------------------------------
self.assertEquals(msg_tuple[1], '*****@*****.**')
#self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS)
#self.assertEquals(message_dict['From'], ION_NOTIFICATION_EMAIL_ADDRESS)
self.assertEquals(message_dict['To'], '*****@*****.**')
self.assertEquals(message_dict['Event'].rstrip('\r'),
'ResourceLifecycleEvent')
self.assertEquals(message_dict['Originator'].rstrip('\r'),
'Some_Resource_Agent_ID1')
self.assertEquals(message_dict['Description'].rstrip('\r'),
'RLE test event')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_sms(self):
proc1 = self.container.proc_manager.procs_by_name['user_notification']
# Create a user and get the user_id
user = UserInfo(name='new_user')
user_id, _ = self.rrc.create(user)
# set up....
notification_id = self.unsc.create_sms(
event_type='ResourceLifecycleEvent',
event_subtype=None,
origin='Some_Resource_Agent_ID1',
origin_type=None,
user_id=user_id,
phone='401-XXX-XXXX',
provider='T-Mobile',
message_header='message_header',
parser='parser',
)
#------------------------------------------------------------------------------------------------------
# Setup so as to be able to get the message and headers going into the
# subscription callback method of the EmailEventProcessor
#------------------------------------------------------------------------------------------------------
# publish an event for each notification to generate the emails
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event")
msg_tuple = proc1.event_processors[
notification_id].smtp_client.sentmail.get(timeout=4)
self.assertTrue(proc1.event_processors[notification_id].smtp_client.
sentmail.empty())
message = msg_tuple[2]
list_lines = message.split("\n")
#-------------------------------------------------------
# parse the message body
#-------------------------------------------------------
message_dict = {}
for line in list_lines:
key_item = line.split(": ")
if key_item[0] == 'Subject':
message_dict['Subject'] = key_item[1] + key_item[2]
else:
try:
message_dict[key_item[0]] = key_item[1]
except IndexError as exc:
# these IndexError exceptions happen only because the message sometimes
# has successive /r/n (i.e. new lines) and therefore,
# the indexing goes out of range. These new lines
# can just be ignored. So we ignore the exceptions here.
pass
#-------------------------------------------------------
# make assertions
#-------------------------------------------------------
self.assertEquals(msg_tuple[1], '*****@*****.**')
#self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS)
self.assertEquals(message_dict['Description'].rstrip('\r'),
'RLE test event')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_event_detection(self):
proc1 = self.container.proc_manager.procs_by_name['user_notification']
# Create a user and get the user_id
user = UserInfo(name='new_user')
user_id, _ = self.rrc.create(user)
# Create detection notification
dfilt = DetectionFilterConfig()
dfilt.processing['condition'] = 5
dfilt.processing['comparator'] = '>'
dfilt.processing['filter_field'] = 'voltage'
dfilt.delivery['message'] = 'I got my detection event!'
notification_id = self.unsc.create_detection_filter(
event_type='ExampleDetectableEvent',
event_subtype=None,
origin='Some_Resource_Agent_ID1',
origin_type=None,
user_id=user_id,
filter_config=dfilt)
#---------------------------------------------------------------------------------
# Create event subscription for resulting detection event
#---------------------------------------------------------------------------------
# Create an email notification so that when the DetectionEventProcessor
# detects an event and fires its own output event, this will caught by an
# EmailEventProcessor and an email will be sent to the user
notification_id_2 = self.unsc.create_email(
event_type='DetectionEvent',
event_subtype=None,
origin='DetectionEventProcessor',
origin_type=None,
user_id=user_id,
email='*****@*****.**',
mode=DeliveryMode.UNFILTERED,
message_header='Detection event',
parser='parser',
period=1)
# Send event that is not detected
# publish an event for each notification to generate the emails
rle_publisher = EventPublisher("ExampleDetectableEvent")
# since the voltage field in this event is less than 5, it will not be detected
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event",
voltage=3)
# Check at the end of the test to make sure this event never triggered a Detectable!
# Send Event that is detected
# publish an event for each notification to generate the emails
# since the voltage field in this event is greater than 5, it WILL be detected
rle_publisher = EventPublisher("ExampleDetectableEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event",
voltage=10)
#-------------------------------------------------------
# make assertions
#-------------------------------------------------------
msg_tuple = proc1.event_processors[
notification_id_2].smtp_client.sentmail.get(timeout=4)
# The first event never triggered an email because the voltage was less than 5, the queue is now empty
self.assertTrue(proc1.event_processors[notification_id_2].smtp_client.
sentmail.empty())
self.assertEquals(msg_tuple[1], '*****@*****.**')
#self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS)
# parse the message body
message = msg_tuple[2]
list_lines = message.split("\n")
message_dict = {}
for line in list_lines:
key_item = line.split(": ")
if key_item[0] == 'Subject':
message_dict['Subject'] = key_item[1] + key_item[2]
else:
try:
message_dict[key_item[0]] = key_item[1]
except IndexError as exc:
# these IndexError exceptions happen only because the message sometimes
# has successive /r/n (i.e. new lines) and therefore,
# the indexing goes out of range. These new lines
# can just be ignored. So we ignore the exceptions here.
pass
#self.assertEquals(message_dict['From'], ION_NOTIFICATION_EMAIL_ADDRESS)
self.assertEquals(message_dict['To'], '*****@*****.**')
self.assertEquals(message_dict['Event'].rstrip('\r'), 'DetectionEvent')
self.assertEquals(message_dict['Originator'].rstrip('\r'),
'DetectionEventProcessor')
self.assertEquals(message_dict['Description'].rstrip('\r'),
'Event was detected by DetectionEventProcessor')
@unittest.skip('interface has changed!')
def test_find_event_types_for_resource(self):
# create a dataset object in the RR to pass into the UNS method
dataset_object = IonObject(RT.DataSet, name="dataset1")
dataset_id, version = self.rrc.create(dataset_object)
# get the list of event types for the dataset
events = self.unsc.find_event_types_for_resource(dataset_id)
log.debug("dataset events = " + str(events))
if not events == ['dataset_supplement_added', 'dataset_change']:
self.fail("failed to return correct list of event types")
# try to pass in an id of a resource that doesn't exist (should fail)
try:
events = self.unsc.find_event_types_for_resource("bogus_id")
self.fail("failed to detect non-existant resource")
except:
pass
@unittest.skip('interface has changed!')
def test_create_two_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(
RT.UserInfo, {
"name": "user1_info",
"contact": {
"email": '*****@*****.**'
}
})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object1 = IonObject(
RT.NotificationRequest, {
"name": "notification1",
"origin_list": ['Some_Resource_Agent_ID1'],
"events_list": ['ResourceLifecycleEvent']
})
notification_id1 = self.unsc.create_notification(
notification_object1, user_id)
# create second notification
notification_object2 = IonObject(
RT.NotificationRequest, {
"name": "notification2",
"origin_list": ['Some_Resource_Agent_ID2'],
"events_list": ['DataEvent']
})
notification_id2 = self.unsc.create_notification(
notification_object2, user_id)
# read the notifications back and check that they are correct
n1 = self.unsc.read_notification(notification_id1)
if n1.name != notification_object1.name or \
n1.origin_list != notification_object1.origin_list or \
n1.events_list != notification_object1.events_list:
self.fail("notification was not correct")
n2 = self.unsc.read_notification(notification_id2)
if n2.name != notification_object2.name or \
n2.origin_list != notification_object2.origin_list or \
n2.events_list != notification_object2.events_list:
self.fail("notification was not correct")
@unittest.skip('interface has changed!')
def test_delete_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(
RT.UserInfo, {
"name": "user1_info",
"contact": {
"email": '*****@*****.**'
}
})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object1 = IonObject(
RT.NotificationRequest, {
"name": "notification1",
"origin_list": ['Some_Resource_Agent_ID1'],
"events_list": ['ResourceLifecycleEvent']
})
notification1_id = self.unsc.create_notification(
notification_object1, user_id)
# create second notification
notification_object2 = IonObject(
RT.NotificationRequest, {
"name": "notification2",
"origin_list": ['Some_Resource_Agent_ID2'],
"events_list": ['DataEvent']
})
notification2_id = self.unsc.create_notification(
notification_object2, user_id)
# delete both notifications
self.unsc.delete_notification(notification1_id)
self.unsc.delete_notification(notification2_id)
# check that the notifications are not there
try:
n1 = self.unsc.read_notification(notification1_id)
except:
try:
n2 = self.unsc.read_notification(notification2_id)
except:
return
self.fail("failed to delete notifications")
@unittest.skip('interface has changed!')
def test_find_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(
RT.UserInfo, {
"name": "user1_info",
"contact": {
"email": '*****@*****.**'
}
})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(
RT.NotificationRequest, {
"name": "notification1",
"origin_list": ['Some_Resource_Agent_ID1'],
"events_list": ['ResourceLifecycleEvent']
})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(
RT.NotificationRequest, {
"name": "notification2",
"origin_list": ['Some_Resource_Agent_ID2'],
"events_list": ['DataEvent']
})
self.unsc.create_notification(notification_object, user_id)
# try to find all notifications for user
notifications = self.unsc.find_notifications_by_user(user_id)
if len(notifications) != 2:
self.fail("failed to find all notifications")
@unittest.skip('interface has changed!')
def test_update_user_notification(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(
RT.UserInfo, {
"name": "user1_info",
"contact": {
"email": '*****@*****.**'
}
})
self.imc.create_user_info(user_id, user_info_object)
# create a notification
notification_object = IonObject(
RT.NotificationRequest, {
"name": "notification1",
"origin_list": ['Some_Resource_Agent_ID1'],
"events_list": ['ResourceLifecycleEvent']
})
notification_id = self.unsc.create_notification(
notification_object, user_id)
# read back the notification and change it
notification = self.unsc.read_notification(notification_id)
notification.origin_list = ['Some_Resource_Agent_ID5']
self.unsc.update_notification(notification)
# read back the notification and check that it got changed
notification = self.unsc.read_notification(notification_id)
if notification.origin_list != ['Some_Resource_Agent_ID5']:
self.fail("failed to change notification")
@unittest.skip('interface has changed!')
def test_send_notification_emails(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {
"name": "user1_info",
"contact": {
"email": '*****@*****.**'
}
})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(
RT.NotificationRequest, {
"name": "notification1",
"origin_list": ['Some_Resource_Agent_ID1'],
"events_list": ['ResourceLifecycleEvent']
})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(
RT.NotificationRequest, {
"name": "notification2",
"origin_list": ['Some_Resource_Agent_ID2'],
"events_list": ['DataEvent']
})
self.unsc.create_notification(notification_object, user_id)
# publish an event for each notification to generate the emails
# this can't be easily check in SW so need to check for these at the [email protected] account
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event")
de_publisher = EventPublisher("DataEvent")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2',
description="DE test event")
gevent.sleep(1)
@unittest.skip('interface has changed!')
def test_find_events(self):
# publish some events for the event repository
rle_publisher = EventPublisher("ResourceLifecycleEvent")
de_publisher = EventPublisher("DataEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event1")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event2")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event3")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2',
description="DE test event1")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2',
description="DE test event2")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2',
description="DE test event3")
# find all events for the originator 'Some_Resource_Agent_ID1'
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1')
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID1':
self.fail("failed to find correct events")
# find all events for the originator 'DataEvent'
events = self.unsc.find_events(type='DataEvent')
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'DataEvent':
self.fail("failed to find correct events")
# find 2 events for the originator 'Some_Resource_Agent_ID1'
events = self.unsc.find_events(origin='Some_Resource_Agent_ID2',
limit=2)
if len(events) != 2:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID2':
self.fail("failed to find correct events")
# find all events for the originator 'Some_Resource_Agent_ID1' in reverse time order
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1',
descending=True)
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID1':
self.fail("failed to find correct events")
class DiscoveryIntTest(IonIntegrationTestCase):
def setUp(self):
super(DiscoveryIntTest, self).setUp()
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
if use_es:
self.es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
self.es_port = CFG.get_safe('server.elasticsearch.port', '9200')
CFG.server.elasticsearch.shards = 1
CFG.server.elasticsearch.replicas = 0
CFG.server.elasticsearch.river_shards = 1
CFG.server.elasticsearch.river_replicas = 0
self.es = ep.ElasticSearch(
host=self.es_host,
port=self.es_port,
timeout=10,
verbose=True
)
op = DotDict(CFG)
op.op = 'clean_bootstrap'
self.container.spawn_process('index_bootstrap','ion.processes.bootstrap.index_bootstrap','IndexBootStrap', op)
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(
host=es_host,
port=es_port,
timeout=10
)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete,index)
IndexManagementService._es_call(es.index_delete,index)
def poll(self, tries, callback, *args, **kwargs):
'''
Polling wrapper for queries
Elasticsearch may not index and cache the changes right away so we may need
a couple of tries and a little time to go by before the results show.
'''
for i in xrange(tries):
retval = callback(*args, **kwargs)
if retval:
return retval
time.sleep(0.2)
return None
def test_traversal(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results = self.discovery.traverse(dp_id)
results.sort()
correct = [pd_id, transform_id]
correct.sort()
self.assertTrue(results == correct, '%s' % results)
def test_iterative_traversal(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results = self.discovery.iterative_traverse(dp_id)
results.sort()
correct = [transform_id]
self.assertTrue(results == correct)
results = self.discovery.iterative_traverse(dp_id, 1)
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@skipIf(not use_es, 'No ElasticSearch')
def test_view_crud(self):
view_id = self.discovery.create_view('big_view',fields=['name'])
catalog_id = self.discovery.list_catalogs(view_id)[0]
index_ids = self.catalog.list_indexes(catalog_id)
self.assertTrue(len(index_ids))
view = self.discovery.read_view(view_id)
self.assertIsInstance(view,View)
self.assertTrue(view.name == 'big_view')
view.name = 'not_so_big_view'
self.discovery.update_view(view)
view = self.discovery.read_view(view_id)
self.assertTrue(view.name == 'not_so_big_view')
self.discovery.delete_view(view_id)
with self.assertRaises(NotFound):
self.discovery.read_view(view_id)
def test_view_best_match(self):
#---------------------------------------------------------------
# Matches the best catalog available OR creates a new one
#---------------------------------------------------------------
catalog_id = self.catalog.create_catalog('dev', keywords=['name','model'])
view_id = self.discovery.create_view('exact_view', fields=['name','model'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids == [catalog_id])
view_id = self.discovery.create_view('another_view', fields=['name','model'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids == [catalog_id])
view_id = self.discovery.create_view('big_view', fields=['name'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids != [catalog_id])
@skipIf(not use_es, 'No ElasticSearch')
def test_basic_searching(self):
#- - - - - - - - - - - - - - - - -
# set up the fake resources
#- - - - - - - - - - - - - - - - -
instrument_pool = [
InstrumentDevice(name='sonobuoy1', hardware_version='1'),
InstrumentDevice(name='sonobuoy2', hardware_version='2'),
InstrumentDevice(name='sonobuoy3', hardware_version='3')
]
for instrument in instrument_pool:
self.rr.create(instrument)
view_id = self.discovery.create_view('devices', fields=['hardware_version'])
search_string = "search 'hardware_version' is '2' from '%s'"%view_id
results = self.poll(5, self.discovery.parse,search_string)
result = results[0]['_source']
self.assertIsInstance(result, InstrumentDevice)
self.assertTrue(result.name == 'sonobuoy2')
self.assertTrue(result.hardware_version == '2')
@skipIf(not use_es, 'No ElasticSearch')
def test_associative_searching(self):
view_id = self.discovery.create_view('devices', fields=['model'])
site_id,_ = self.rr.create(Site('my_site'))
pd_id, _ = self.rr.create(PlatformDevice('my_device', model='abc123'))
self.rr.create_association(subject=site_id, object=pd_id, predicate=PRED.hasDevice)
search_string = "search 'model' is 'abc*' from '%s' and belongs to '%s'"%(view_id, site_id)
results = self.poll(5, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(pd_id in results)
def test_iterative_associative_searching(self):
#--------------------------------------------------------------------------------
# Tests the ability to limit the iterations
#--------------------------------------------------------------------------------
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
search_string = "belongs to '%s' depth 1" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
correct = [transform_id]
self.assertTrue(results == correct, '%s' % results)
search_string = "belongs to '%s' depth 2" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@skipIf(not use_es, 'No ElasticSearch')
def test_ranged_value_searching(self):
discovery = self.discovery
rr = self.rr
view_id = discovery.create_view('bank_view', fields=['cash_balance'])
bank_id, _ = rr.create(BankAccount(name='broke', cash_balance=10))
search_string = "search 'cash_balance' values from 0 to 100 from '%s'" % view_id
results = self.poll(5, discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == bank_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_collections_searching(self):
site_id, _ = self.rr.create(Site(name='black_mesa'))
view_id = self.discovery.create_view('big', fields=['name'])
# Add the site to a new collection
collection_id = self.ims.create_collection('resource_collection', [site_id])
search_string = "search 'name' is '*' from '%s' and in '%s'" %(view_id, collection_id)
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0] == site_id, '%s' % results)
@skipIf(not use_es, 'No ElasticSearch')
def test_search_by_name(self):
inst_dev = InstrumentDevice(name='test_dev',serial_number='ABC123')
dev_id, _ = self.rr.create(inst_dev)
self.discovery.create_view('devs',fields=['name','serial_number'])
search_string = "search 'serial_number' is 'abc*' from 'devs'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dev_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_search_by_name_index(self):
inst_dev = InstrumentDevice(name='test_dev',serial_number='ABC123')
dev_id, _ = self.rr.create(inst_dev)
search_string = "search 'serial_number' is 'abc*' from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dev_id)
bank_acc = BankAccount(name='blah', cash_balance=10)
res_id , _ = self.rr.create(bank_acc)
search_string = "search 'cash_balance' values from 0 to 100 from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == res_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_data_product_search(self):
# Create the dataproduct
dp = DataProduct(name='test_product')
dp.data_format.name = 'test_signal'
dp.data_format.description = 'test signal'
dp.data_format.character_set = 'utf8'
dp.data_format.nominal_sampling_rate_maximum = '44000'
dp.data_format.nominal_sampling_rate_minimum = '44000'
dp.data_product_level = 'basic'
dp_id, _ = self.rr.create(dp)
search_string = "search 'data_format.name' is 'test_signal' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'data_product_level' is 'basic' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'data_format.character_set' is 'utf8' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_events_search(self):
# Create a resource to force a new event
dp = DataProcess()
dp_id, rev = self.rr.create(dp)
search_string = "SEARCH 'origin' IS '%s' FROM 'events_index'" % dp_id
results = self.poll(9, self.discovery.parse,search_string)
origin_type = results[0]['_source'].origin_type
origin_id = results[0]['_source'].origin
self.assertTrue(origin_type == RT.DataProcess)
self.assertTrue(origin_id == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_geo_distance_search(self):
pd = PlatformDevice(name='test_dev')
pd_id, _ = self.rr.create(pd)
search_string = "search 'nominal_location' geo distance 20 km from lat 0 lon 0 from 'devices_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == pd_id)
self.assertTrue(results[0]['_source'].name == 'test_dev')
@skipIf(not use_es, 'No ElasticSearch')
def test_geo_bbox_search(self):
pd = PlatformDevice(name='test_dev')
pd.nominal_location.lat = 5
pd.nominal_location.lon = 5
pd_id, _ = self.rr.create(pd)
search_string = "search 'nominal_location' geo box top-left lat 10 lon 0 bottom-right lat 0 lon 10 from 'devices_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == pd_id)
self.assertTrue(results[0]['_source'].name == 'test_dev')
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(TestActivateInstrumentIntegration.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete, index)
IndexManagementService._es_call(es.index_delete, index)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_notification(self,
user_name='',
instrument_id='',
product_id=''):
#--------------------------------------------------------------------------------------
# Make notification request objects
#--------------------------------------------------------------------------------------
notification_request_1 = NotificationRequest(
name='notification_1',
origin=instrument_id,
origin_type="instrument",
event_type='ResourceLifecycleEvent')
notification_request_2 = NotificationRequest(
name='notification_2',
origin=product_id,
origin_type="data product",
event_type='DetectionEvent')
#--------------------------------------------------------------------------------------
# Create a user and get the user_id
#--------------------------------------------------------------------------------------
user = UserInfo()
user.name = user_name
user.contact.email = '*****@*****.**' % user_name
user_id, _ = self.rrclient.create(user)
#--------------------------------------------------------------------------------------
# Create notification
#--------------------------------------------------------------------------------------
self.usernotificationclient.create_notification(
notification=notification_request_1, user_id=user_id)
self.usernotificationclient.create_notification(
notification=notification_request_2, user_id=user_id)
log.debug(
"test_activateInstrumentSample: create_user_notifications user_id %s",
str(user_id))
return user_id
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def _check_computed_attributes_of_extended_instrument(
self, expected_instrument_device_id='', extended_instrument=None):
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version,
ComputedFloatValue)
self.assertIsInstance(
extended_instrument.computed.last_data_received_datetime,
ComputedFloatValue)
self.assertIsInstance(
extended_instrument.computed.last_calibration_datetime,
ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
self.assertIsInstance(
extended_instrument.computed.power_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.communications_status_roll_up,
ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.location_status_roll_up,
ComputedIntValue)
# the following assert will not work without elasticsearch.
#self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
self.assertEqual(
extended_instrument.computed.communications_status_roll_up.value,
StatusType.STATUS_WARNING)
self.assertEqual(
extended_instrument.computed.data_status_roll_up.value,
StatusType.STATUS_OK)
self.assertEqual(
extended_instrument.computed.power_status_roll_up.value,
StatusType.STATUS_WARNING)
# Verify the computed attribute for user notification requests
self.assertEqual(
1,
len(extended_instrument.computed.user_notification_requests.value))
notifications = extended_instrument.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_instrument_device_id)
self.assertEqual(notification.origin_type, "instrument")
self.assertEqual(notification.event_type, 'ResourceLifecycleEvent')
def _check_computed_attributes_of_extended_product(
self, expected_data_product_id='', extended_data_product=None):
self.assertEqual(expected_data_product_id, extended_data_product._id)
log.debug("extended_data_product.computed: %s",
extended_data_product.computed)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(
extended_data_product.computed.product_download_size_estimated,
ComputedIntValue)
self.assertIsInstance(
extended_data_product.computed.number_active_subscriptions,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_url,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.stored_data_size,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.recent_granules,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.parameters,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.recent_events,
ComputedEventListValue)
self.assertIsInstance(extended_data_product.computed.provenance,
ComputedDictValue)
self.assertIsInstance(
extended_data_product.computed.user_notification_requests,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.active_user_subscriptions,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.past_user_subscriptions,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.last_granule,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.is_persisted,
ComputedIntValue)
self.assertIsInstance(
extended_data_product.computed.data_contents_updated,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.data_datetime,
ComputedListValue)
# exact text here keeps changing to fit UI capabilities. keep assertion general...
self.assertTrue('ok' in extended_data_product.computed.last_granule.
value['quality_flag'])
self.assertEqual(
2, len(extended_data_product.computed.data_datetime.value))
notifications = extended_data_product.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_data_product_id)
self.assertEqual(notification.origin_type, "data product")
self.assertEqual(notification.event_type, 'DetectionEvent')
@attr('LOCOINT')
@unittest.skipIf(not use_es, 'No ElasticSearch')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}})
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', instModel_id)
#Create stream alarms
"""
test_two_sided_interval
Test interval alarm and alarm event publishing for a closed
inteval.
"""
# kwargs = {
# 'name' : 'test_sim_warning',
# 'stream_name' : 'parsed',
# 'value_id' : 'temp',
# 'message' : 'Temperature is above test range of 5.0.',
# 'type' : StreamAlarmType.WARNING,
# 'upper_bound' : 5.0,
# 'upper_rel_op' : '<'
# }
kwargs = {
'name': 'temperature_warning_interval',
'stream_name': 'parsed',
'value_id': 'temp',
'message':
'Temperature is below the normal range of 50.0 and above.',
'type': StreamAlarmType.WARNING,
'lower_bound': 50.0,
'lower_rel_op': '<'
}
# Create alarm object.
alarm = {}
alarm['type'] = 'IntervalAlarmDef'
alarm['kwargs'] = kwargs
raw_config = StreamConfiguration(
stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5)
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict',
records_per_granule=2,
granule_publish_rate=5,
alarms=[alarm])
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=
"http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg",
stream_configurations=[raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0])
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# setup notifications for the device and parsed data product
user_id_1 = self._create_notification(user_name='user_1',
instrument_id=instDevice_id,
product_id=data_product_id1)
#---------- Create notifications for another user and verify that we see different computed subscriptions for the two users ---------
user_id_2 = self._create_notification(user_name='user_2',
instrument_id=instDevice_id,
product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
log.debug('Data product streams2 = %s', str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id2 = %s', dataset_ids[0])
self.raw_dataset = dataset_ids[0]
#elastic search debug
es_indexes, _ = self.container.resource_registry.find_resources(
restype='ElasticSearchIndex')
log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes])
log.debug('Bootstrap %s', CFG.bootstrap.use_es)
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#setup a subscriber to alarm events from the device
self._events_received = []
self._event_count = 0
self._samples_out_of_range = 0
self._samples_complete = False
self._async_sample_result = AsyncResult()
def consume_event(*args, **kwargs):
log.debug(
'TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
self._event_count = len(self._events_received)
self._async_sample_result.set()
self._event_subscriber = EventSubscriber(
event_type=
'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent
callback=consume_event,
origin=instDevice_id)
self._event_subscriber.start()
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
def stop_subscriber():
self._event_subscriber.stop()
self._event_subscriber = None
self.addCleanup(stop_subscriber)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
inst_agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
inst_agent_instance_obj.agent_process_id)
log.debug('Instrument agent instance obj: = %s',
str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(
instDevice_id,
to_name=inst_agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s",
str(self._ia_client))
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s", str(retval))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s",
str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active command %s",
str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s", str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
for i in xrange(10):
retval = self._ia_client.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from sample %s",
str(retval))
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s",
str(reply))
self._samples_complete = True
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("test_activateInstrumentSample: RDT parsed: %s",
str(rdt.pretty_print()))
temp_vals = rdt['temp']
self.assertEquals(len(temp_vals), 10)
log.debug("test_activateInstrumentSample: all temp_vals: %s",
temp_vals)
#out_of_range_temp_vals = [i for i in temp_vals if i > 5]
out_of_range_temp_vals = [i for i in temp_vals if i < 50.0]
log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s",
out_of_range_temp_vals)
self._samples_out_of_range = len(out_of_range_temp_vals)
# if no bad values were produced, then do not wait for an event
if self._samples_out_of_range == 0:
self._async_sample_result.set()
log.debug("test_activateInstrumentSample: _events_received: %s",
self._events_received)
log.debug("test_activateInstrumentSample: _event_count: %s",
self._event_count)
self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout)
replay_data = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("RDT raw: %s", str(rdt.pretty_print()))
raw_vals = rdt['raw']
self.assertEquals(len(raw_vals), 10)
log.debug("l4-ci-sa-rq-138")
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
#--------------------------------------------------------------------------------
# Get the extended data product to see if it contains the granules
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id1, user_id=user_id_1)
def poller(extended_product):
return len(extended_product.computed.user_notification_requests.
value) == 1
poll(poller, extended_product, timeout=30)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id1,
extended_data_product=extended_product)
#--------------------------------------------------------------------------------
#put some events into the eventsdb to test - this should set the comms and data status to WARNING
#--------------------------------------------------------------------------------
t = get_ion_ts()
self.event_publisher.publish_event(ts_created=t,
event_type='DeviceStatusEvent',
origin=instDevice_id,
state=DeviceStatusType.OUT_OF_RANGE,
values=[200])
self.event_publisher.publish_event(
ts_created=t,
event_type='DeviceCommsEvent',
origin=instDevice_id,
state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION,
lapse_interval_seconds=20)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_1)
self._check_computed_attributes_of_extended_instrument(
expected_instrument_device_id=instDevice_id,
extended_instrument=extended_instrument)
#--------------------------------------------------------------------------------
# For the second user, check the extended data product and the extended intrument
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id2, user_id=user_id_2)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id2,
extended_data_product=extended_product)
#---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING ---------
t = get_ion_ts()
self.event_publisher.publish_event(ts_created=t,
event_type='DeviceStatusEvent',
origin=instDevice_id,
state=DeviceStatusType.OUT_OF_RANGE,
values=[200])
self.event_publisher.publish_event(
ts_created=t,
event_type='DeviceCommsEvent',
origin=instDevice_id,
state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION,
lapse_interval_seconds=20)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_2)
self._check_computed_attributes_of_extended_instrument(
expected_instrument_device_id=instDevice_id,
extended_instrument=extended_instrument)
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(data_product_id1)
self.dpclient.delete_data_product(data_product_id2)
class TestObservatoryManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.org_management_service = OrgManagementServiceClient(
node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#print 'TestObservatoryManagementServiceIntegration: started services'
self.event_publisher = EventPublisher()
# @unittest.skip('this exists only for debugging the launch process')
# def test_just_the_setup(self):
# return
def destroy(self, resource_ids):
self.OMS.force_delete_observatory(resource_ids.observatory_id)
self.OMS.force_delete_subsite(resource_ids.subsite_id)
self.OMS.force_delete_subsite(resource_ids.subsite2_id)
self.OMS.force_delete_subsite(resource_ids.subsiteb_id)
self.OMS.force_delete_subsite(resource_ids.subsitez_id)
self.OMS.force_delete_platform_site(resource_ids.platform_site_id)
self.OMS.force_delete_platform_site(resource_ids.platform_siteb_id)
self.OMS.force_delete_platform_site(resource_ids.platform_siteb2_id)
self.OMS.force_delete_platform_site(resource_ids.platform_site3_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_site_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_site2_id)
self.OMS.force_delete_instrument_site(
resource_ids.instrument_siteb3_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_site4_id)
#@unittest.skip('targeting')
def test_observatory_management(self):
resources = self._make_associations()
self._do_test_find_related_sites(resources)
self._do_test_get_sites_devices_status(resources)
self._do_test_find_site_data_products(resources)
self._do_test_find_related_frames_of_reference(resources)
self._do_test_create_geospatial_point_center(resources)
self._do_test_find_observatory_org(resources)
self.destroy(resources)
def _do_test_find_related_sites(self, resources):
site_resources, site_children, _, _ = self.OMS.find_related_sites(
resources.org_id)
#import sys, pprint
#print >> sys.stderr, pprint.pformat(site_resources)
#print >> sys.stderr, pprint.pformat(site_children)
#self.assertIn(resources.org_id, site_resources)
self.assertIn(resources.observatory_id, site_resources)
self.assertIn(resources.subsite_id, site_resources)
self.assertIn(resources.subsite_id, site_resources)
self.assertIn(resources.subsite2_id, site_resources)
self.assertIn(resources.platform_site_id, site_resources)
self.assertIn(resources.instrument_site_id, site_resources)
self.assertEquals(len(site_resources), 13)
self.assertEquals(site_resources[resources.observatory_id].type_,
RT.Observatory)
self.assertIn(resources.org_id, site_children)
self.assertIn(resources.observatory_id, site_children)
self.assertIn(resources.subsite_id, site_children)
self.assertIn(resources.subsite_id, site_children)
self.assertIn(resources.subsite2_id, site_children)
self.assertIn(resources.platform_site_id, site_children)
self.assertNotIn(resources.instrument_site_id, site_children)
self.assertEquals(len(site_children), 9)
self.assertIsInstance(site_children[resources.subsite_id], list)
self.assertEquals(len(site_children[resources.subsite_id]), 2)
def _do_test_get_sites_devices_status(self, resources):
#bin/nosetests -s -v --nologcapture ion/services/sa/observatory/test/test_observatory_management_service_integration.py:TestObservatoryManagementServiceIntegration.test_observatory_management
full_result_dict = self.OMS.get_sites_devices_status(
parent_resource_ids=[resources.org_id], include_sites=True)
result_dict = full_result_dict[resources.org_id]
site_resources = result_dict.get("site_resources", None)
site_children = result_dict.get("site_children", None)
self.assertEquals(len(site_resources), 14)
self.assertEquals(len(site_children), 9)
full_result_dict = self.OMS.get_sites_devices_status(
parent_resource_ids=[resources.org_id],
include_sites=True,
include_devices=True,
include_status=True)
result_dict = full_result_dict[resources.org_id]
log.debug("RESULT DICT: %s", result_dict.keys())
site_resources = result_dict.get("site_resources", None)
site_children = result_dict.get("site_children", None)
site_status = result_dict.get("site_status", None)
self.assertEquals(len(site_resources), 14)
self.assertEquals(len(site_children), 9)
full_result_dict = self.OMS.get_sites_devices_status(
parent_resource_ids=[resources.observatory_id],
include_sites=True,
include_devices=True,
include_status=True)
result_dict = full_result_dict[resources.observatory_id]
site_resources = result_dict.get("site_resources")
site_children = result_dict.get("site_children")
site_status = result_dict.get("site_status")
self.assertEquals(len(site_resources), 13)
self.assertEquals(len(site_children), 8)
def _do_test_find_site_data_products(self, resources):
res_dict = self.OMS.find_site_data_products(resources.org_id)
#import sys, pprint
#print >> sys.stderr, pprint.pformat(res_dict)
self.assertIsNone(res_dict['data_product_resources'])
self.assertIn(resources.platform_device_id,
res_dict['device_data_products'])
self.assertIn(resources.instrument_device_id,
res_dict['device_data_products'])
#@unittest.skip('targeting')
def _do_test_find_related_frames_of_reference(self, stuff):
# finding subordinates gives a dict of obj lists, convert objs to ids
def idify(adict):
ids = {}
for k, v in adict.iteritems():
ids[k] = []
for obj in v:
ids[k].append(obj._id)
return ids
# a short version of the function we're testing, with id-ify
def short(resource_id, output_types):
ret = self.OMS.find_related_frames_of_reference(
resource_id, output_types)
return idify(ret)
#set up associations first
stuff = self._make_associations()
#basic traversal of tree from instrument to platform
ids = short(stuff.instrument_site_id, [RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
#since this is the first search, just make sure the input inst_id got stripped
if RT.InstrumentSite in ids:
self.assertNotIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
#basic traversal of tree from platform to instrument
ids = short(stuff.platform_siteb_id, [RT.InstrumentSite])
self.assertIn(RT.InstrumentSite, ids)
self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
self.assertNotIn(stuff.instrument_site2_id, ids[RT.InstrumentSite])
#full traversal of tree from observatory down to instrument
ids = short(stuff.observatory_id, [RT.InstrumentSite])
self.assertIn(RT.InstrumentSite, ids)
self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
#full traversal of tree from instrument to observatory
ids = short(stuff.instrument_site_id, [RT.Observatory])
self.assertIn(RT.Observatory, ids)
self.assertIn(stuff.observatory_id, ids[RT.Observatory])
#partial traversal, only down to platform
ids = short(stuff.observatory_id, [RT.Subsite, RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(RT.Subsite, ids)
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_site3_id, ids[RT.PlatformSite])
self.assertIn(stuff.subsite_id, ids[RT.Subsite])
self.assertIn(stuff.subsite2_id, ids[RT.Subsite])
self.assertIn(stuff.subsitez_id, ids[RT.Subsite])
self.assertIn(stuff.subsiteb_id, ids[RT.Subsite])
self.assertNotIn(RT.InstrumentSite, ids)
#partial traversal, only down to platform
ids = short(stuff.instrument_site_id, [RT.Subsite, RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(RT.Subsite, ids)
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.subsite_id, ids[RT.Subsite])
self.assertIn(stuff.subsiteb_id, ids[RT.Subsite])
self.assertNotIn(stuff.subsite2_id, ids[RT.Subsite])
self.assertNotIn(stuff.subsitez_id, ids[RT.Subsite])
self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
self.assertNotIn(RT.Observatory, ids)
self.destroy(stuff)
def _make_associations(self):
"""
create one of each resource and association used by OMS
to guard against problems in ion-definitions
"""
#raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41")
"""
the tree we're creating (observatory, sites, platforms, instruments)
rows are lettered, colums numbered.
- first row is implied a
- first column is implied 1
- site Z, just because
O--Sz
|
S--S2--P3--I4
|
Sb-Pb2-Ib3
|
P--I2 <- PlatformDevice, InstrumentDevice2
|
Pb <- PlatformDevice b
|
I <- InstrumentDevice
"""
org_id = self.OMS.create_marine_facility(any_old(RT.Org))
def create_under_org(resource_type, extra_fields=None):
obj = any_old(resource_type, extra_fields)
if RT.InstrumentDevice == resource_type:
resource_id = self.IMS.create_instrument_device(obj)
else:
resource_id, _ = self.RR.create(obj)
self.OMS.assign_resource_to_observatory_org(
resource_id=resource_id, org_id=org_id)
return resource_id
#stuff we control
observatory_id = create_under_org(RT.Observatory)
subsite_id = create_under_org(RT.Subsite)
subsite2_id = create_under_org(RT.Subsite)
subsiteb_id = create_under_org(RT.Subsite)
subsitez_id = create_under_org(RT.Subsite)
platform_site_id = create_under_org(RT.PlatformSite)
platform_siteb_id = create_under_org(RT.PlatformSite)
platform_siteb2_id = create_under_org(RT.PlatformSite)
platform_site3_id = create_under_org(RT.PlatformSite)
instrument_site_id = create_under_org(RT.InstrumentSite)
instrument_site2_id = create_under_org(RT.InstrumentSite)
instrument_siteb3_id = create_under_org(RT.InstrumentSite)
instrument_site4_id = create_under_org(RT.InstrumentSite)
#stuff we associate to
instrument_device_id = create_under_org(RT.InstrumentDevice)
instrument_device2_id = create_under_org(RT.InstrumentDevice)
platform_device_id = create_under_org(RT.PlatformDevice)
platform_deviceb_id = create_under_org(RT.PlatformDevice)
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
deployment_id, _ = self.RR.create(any_old(RT.Deployment))
# marine tracking resources
asset_id = create_under_org(RT.Asset)
asset_type_id = create_under_org(RT.AssetType)
event_duration_id = create_under_org(RT.EventDuration)
event_duration_type_id = create_under_org(RT.EventDurationType)
#observatory
self.RR.create_association(observatory_id, PRED.hasSite, subsite_id)
self.RR.create_association(observatory_id, PRED.hasSite, subsitez_id)
#site
self.RR.create_association(subsite_id, PRED.hasSite, subsite2_id)
self.RR.create_association(subsite_id, PRED.hasSite, subsiteb_id)
self.RR.create_association(subsite2_id, PRED.hasSite,
platform_site3_id)
self.RR.create_association(subsiteb_id, PRED.hasSite,
platform_siteb2_id)
self.RR.create_association(subsiteb_id, PRED.hasSite, platform_site_id)
#platform_site(s)
self.RR.create_association(platform_site3_id, PRED.hasSite,
instrument_site4_id)
self.RR.create_association(platform_siteb2_id, PRED.hasSite,
instrument_siteb3_id)
self.RR.create_association(platform_site_id, PRED.hasSite,
instrument_site2_id)
self.RR.create_association(platform_site_id, PRED.hasSite,
platform_siteb_id)
self.RR.create_association(platform_siteb_id, PRED.hasSite,
instrument_site_id)
self.RR.create_association(platform_siteb_id, PRED.hasDevice,
platform_deviceb_id)
#test network parent link
self.OMS.assign_device_to_network_parent(platform_device_id,
platform_deviceb_id)
self.RR.create_association(platform_site_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_site_id, PRED.hasDevice,
platform_device_id)
self.RR.create_association(platform_site_id, PRED.hasDeployment,
deployment_id)
#instrument_site(s)
self.RR.create_association(instrument_site_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(instrument_site_id, PRED.hasDevice,
instrument_device_id)
self.RR.create_association(instrument_site_id, PRED.hasDeployment,
deployment_id)
self.RR.create_association(instrument_site2_id, PRED.hasDevice,
instrument_device2_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel,
platform_model_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(instrument_device2_id, PRED.hasModel,
instrument_model_id)
ret = DotDict()
ret.org_id = org_id
ret.observatory_id = observatory_id
ret.subsite_id = subsite_id
ret.subsite2_id = subsite2_id
ret.subsiteb_id = subsiteb_id
ret.subsitez_id = subsitez_id
ret.platform_site_id = platform_site_id
ret.platform_siteb_id = platform_siteb_id
ret.platform_siteb2_id = platform_siteb2_id
ret.platform_site3_id = platform_site3_id
ret.instrument_site_id = instrument_site_id
ret.instrument_site2_id = instrument_site2_id
ret.instrument_siteb3_id = instrument_siteb3_id
ret.instrument_site4_id = instrument_site4_id
ret.instrument_device_id = instrument_device_id
ret.instrument_device2_id = instrument_device2_id
ret.platform_device_id = platform_device_id
ret.platform_deviceb_id = platform_deviceb_id
ret.instrument_model_id = instrument_model_id
ret.platform_model_id = platform_model_id
ret.deployment_id = deployment_id
ret.asset_id = asset_id
ret.asset_type_id = asset_type_id
ret.event_duration_id = event_duration_id
ret.event_duration_type_id = event_duration_type_id
return ret
#@unittest.skip("targeting")
def test_create_observatory(self):
observatory_obj = IonObject(RT.Observatory,
name='TestFacility',
description='some new mf')
observatory_id = self.OMS.create_observatory(observatory_obj)
self.OMS.force_delete_observatory(observatory_id)
#@unittest.skip("targeting")
def _do_test_create_geospatial_point_center(self, resources):
platformsite_obj = IonObject(RT.PlatformSite,
name='TestPlatformSite',
description='some new TestPlatformSite')
geo_index_obj = IonObject(OT.GeospatialBounds)
geo_index_obj.geospatial_latitude_limit_north = 20.0
geo_index_obj.geospatial_latitude_limit_south = 10.0
geo_index_obj.geospatial_longitude_limit_east = 15.0
geo_index_obj.geospatial_longitude_limit_west = 20.0
platformsite_obj.constraint_list = [geo_index_obj]
platformsite_id = self.OMS.create_platform_site(platformsite_obj)
# now get the dp back to see if it was updated
platformsite_obj = self.OMS.read_platform_site(platformsite_id)
self.assertEquals('some new TestPlatformSite',
platformsite_obj.description)
self.assertAlmostEqual(15.0,
platformsite_obj.geospatial_point_center.lat,
places=1)
#now adjust a few params
platformsite_obj.description = 'some old TestPlatformSite'
geo_index_obj = IonObject(OT.GeospatialBounds)
geo_index_obj.geospatial_latitude_limit_north = 30.0
geo_index_obj.geospatial_latitude_limit_south = 20.0
platformsite_obj.constraint_list = [geo_index_obj]
update_result = self.OMS.update_platform_site(platformsite_obj)
# now get the dp back to see if it was updated
platformsite_obj = self.OMS.read_platform_site(platformsite_id)
self.assertEquals('some old TestPlatformSite',
platformsite_obj.description)
self.assertAlmostEqual(25.0,
platformsite_obj.geospatial_point_center.lat,
places=1)
self.OMS.force_delete_platform_site(platformsite_id)
#@unittest.skip("targeting")
def _do_test_find_observatory_org(self, resources):
log.debug("Make TestOrg")
org_obj = IonObject(RT.Org,
name='TestOrg',
description='some new mf org')
org_id = self.OMS.create_marine_facility(org_obj)
log.debug("Make Observatory")
observatory_obj = IonObject(RT.Observatory,
name='TestObservatory',
description='some new obs')
observatory_id = self.OMS.create_observatory(observatory_obj)
log.debug("assign observatory to org")
self.OMS.assign_resource_to_observatory_org(observatory_id, org_id)
log.debug("verify assigment")
org_objs = self.OMS.find_org_by_observatory(observatory_id)
self.assertEqual(1, len(org_objs))
self.assertEqual(org_id, org_objs[0]._id)
log.debug("org_id=<" + org_id + ">")
log.debug("create a subsite with parent Observatory")
subsite_obj = IonObject(RT.Subsite,
name='TestSubsite',
description='sample subsite')
subsite_id = self.OMS.create_subsite(subsite_obj, observatory_id)
self.assertIsNotNone(subsite_id, "Subsite not created.")
log.debug("verify that Subsite is linked to Observatory")
mf_subsite_assoc = self.RR.get_association(observatory_id,
PRED.hasSite, subsite_id)
self.assertIsNotNone(mf_subsite_assoc,
"Subsite not connected to Observatory.")
log.debug("add the Subsite as a resource of this Observatory")
self.OMS.assign_resource_to_observatory_org(resource_id=subsite_id,
org_id=org_id)
log.debug("verify that Subsite is linked to Org")
org_subsite_assoc = self.RR.get_association(org_id, PRED.hasResource,
subsite_id)
self.assertIsNotNone(org_subsite_assoc,
"Subsite not connected as resource to Org.")
log.debug("create a logical platform with parent Subsite")
platform_site_obj = IonObject(RT.PlatformSite,
name='TestPlatformSite',
description='sample logical platform')
platform_site_id = self.OMS.create_platform_site(
platform_site_obj, subsite_id)
self.assertIsNotNone(platform_site_id, "PlatformSite not created.")
log.debug("verify that PlatformSite is linked to Site")
site_lp_assoc = self.RR.get_association(subsite_id, PRED.hasSite,
platform_site_id)
self.assertIsNotNone(site_lp_assoc,
"PlatformSite not connected to Site.")
log.debug("add the PlatformSite as a resource of this Observatory")
self.OMS.assign_resource_to_observatory_org(
resource_id=platform_site_id, org_id=org_id)
log.debug("verify that PlatformSite is linked to Org")
org_lp_assoc = self.RR.get_association(org_id, PRED.hasResource,
platform_site_id)
self.assertIsNotNone(org_lp_assoc,
"PlatformSite not connected as resource to Org.")
log.debug("create a logical instrument with parent logical platform")
instrument_site_obj = IonObject(
RT.InstrumentSite,
name='TestInstrumentSite',
description='sample logical instrument')
instrument_site_id = self.OMS.create_instrument_site(
instrument_site_obj, platform_site_id)
self.assertIsNotNone(instrument_site_id, "InstrumentSite not created.")
log.debug("verify that InstrumentSite is linked to PlatformSite")
li_lp_assoc = self.RR.get_association(platform_site_id, PRED.hasSite,
instrument_site_id)
self.assertIsNotNone(li_lp_assoc,
"InstrumentSite not connected to PlatformSite.")
log.debug("add the InstrumentSite as a resource of this Observatory")
self.OMS.assign_resource_to_observatory_org(
resource_id=instrument_site_id, org_id=org_id)
log.debug("verify that InstrumentSite is linked to Org")
org_li_assoc = self.RR.get_association(org_id, PRED.hasResource,
instrument_site_id)
self.assertIsNotNone(
org_li_assoc, "InstrumentSite not connected as resource to Org.")
log.debug(
"remove the InstrumentSite as a resource of this Observatory")
self.OMS.unassign_resource_from_observatory_org(
instrument_site_id, org_id)
log.debug("verify that InstrumentSite is linked to Org")
assocs, _ = self.RR.find_objects(org_id,
PRED.hasResource,
RT.InstrumentSite,
id_only=True)
self.assertEqual(0, len(assocs))
log.debug(
"remove the InstrumentSite, association should drop automatically")
self.OMS.delete_instrument_site(instrument_site_id)
assocs, _ = self.RR.find_objects(platform_site_id,
PRED.hasSite,
RT.InstrumentSite,
id_only=True)
self.assertEqual(0, len(assocs))
log.debug("remove the PlatformSite as a resource of this Observatory")
self.OMS.unassign_resource_from_observatory_org(
platform_site_id, org_id)
log.debug("verify that PlatformSite is linked to Org")
assocs, _ = self.RR.find_objects(org_id,
PRED.hasResource,
RT.PlatformSite,
id_only=True)
self.assertEqual(0, len(assocs))
log.debug("remove the Site as a resource of this Observatory")
self.OMS.unassign_resource_from_observatory_org(subsite_id, org_id)
log.debug("verify that Site is linked to Org")
assocs, _ = self.RR.find_objects(org_id,
PRED.hasResource,
RT.Subsite,
id_only=True)
self.assertEqual(0, len(assocs))
self.RR.delete(org_id)
self.OMS.force_delete_observatory(observatory_id)
self.OMS.force_delete_subsite(subsite_id)
self.OMS.force_delete_platform_site(platform_site_id)
self.OMS.force_delete_instrument_site(instrument_site_id)
@attr('EXT')
@unittest.skipIf(os.getenv(
'CEI_LAUNCH_TEST', False
), 'Skip test while in CEI LAUNCH mode as it depends on modifying CFG on service side'
)
def test_observatory_extensions(self):
self.patch_cfg(CFG["container"],
{"extended_resources": {
"strip_results": False
}})
obs_id = self.RR2.create(any_old(RT.Observatory))
pss_id = self.RR2.create(
any_old(RT.PlatformSite, dict(alt_resource_type="StationSite")))
pas_id = self.RR2.create(
any_old(RT.PlatformSite,
dict(alt_resource_type="PlatformAssemblySite")))
pcs_id = self.RR2.create(
any_old(RT.PlatformSite,
dict(alt_resource_type="PlatformComponentSite")))
ins_id = self.RR2.create(any_old(RT.InstrumentSite))
obs_obj = self.RR2.read(obs_id)
pss_obj = self.RR2.read(pss_id)
pas_obj = self.RR2.read(pas_id)
pcs_obj = self.RR2.read(pcs_id)
ins_obj = self.RR2.read(ins_id)
self.RR2.create_association(obs_id, PRED.hasSite, pss_id)
self.RR2.create_association(pss_id, PRED.hasSite, pas_id)
self.RR2.create_association(pas_id, PRED.hasSite, pcs_id)
self.RR2.create_association(pcs_id, PRED.hasSite, ins_id)
extended_obs = self.OMS.get_observatory_site_extension(obs_id,
user_id=12345)
self.assertEqual([pss_obj], extended_obs.platform_station_sites)
self.assertEqual([pas_obj], extended_obs.platform_assembly_sites)
self.assertEqual([pcs_obj], extended_obs.platform_component_sites)
self.assertEqual([ins_obj], extended_obs.instrument_sites)
extended_pss = self.OMS.get_observatory_site_extension(obs_id,
user_id=12345)
self.assertEqual([pas_obj], extended_pss.platform_assembly_sites)
self.assertEqual([pcs_obj], extended_pss.platform_component_sites)
self.assertEqual([ins_obj], extended_pss.instrument_sites)
extended_pas = self.OMS.get_observatory_site_extension(pas_id,
user_id=12345)
self.assertEqual([pcs_obj], extended_pas.platform_component_sites)
self.assertEqual([ins_obj], extended_pas.instrument_sites)
extended_pcs = self.OMS.get_platform_component_site_extension(
pcs_id, user_id=12345)
self.assertEqual([ins_obj], extended_pcs.instrument_sites)
#@unittest.skip("in development...")
@attr('EXT')
@attr('EXT1')
@unittest.skipIf(os.getenv(
'CEI_LAUNCH_TEST', False
), 'Skip test while in CEI LAUNCH mode as it depends on modifying CFG on service side'
)
def test_observatory_org_extended(self):
self.patch_cfg(CFG["container"],
{"extended_resources": {
"strip_results": False
}})
stuff = self._make_associations()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test')
data_product_id1 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=stuff.instrument_device_id,
data_product_id=data_product_id1)
#Create a user to be used as regular member
member_actor_obj = IonObject(RT.ActorIdentity, name='org member actor')
member_actor_id, _ = self.RR.create(member_actor_obj)
assert (member_actor_id)
member_actor_header = get_actor_header(member_actor_id)
member_user_obj = IonObject(RT.UserInfo, name='org member user')
member_user_id, _ = self.RR.create(member_user_obj)
assert (member_user_id)
self.RR.create_association(subject=member_actor_id,
predicate=PRED.hasInfo,
object=member_user_id)
#Build the Service Agreement Proposal to enroll a user actor
sap = IonObject(OT.EnrollmentProposal,
consumer=member_actor_id,
provider=stuff.org_id)
sap_response = self.org_management_service.negotiate(
sap, headers=member_actor_header)
#enroll the member without using negotiation
self.org_management_service.enroll_member(org_id=stuff.org_id,
actor_id=member_actor_id)
#--------------------------------------------------------------------------------
# Get the extended Site (platformSite)
#--------------------------------------------------------------------------------
try:
extended_site = self.OMS.get_site_extension(stuff.platform_site_id)
except:
log.error('failed to get extended site', exc_info=True)
raise
log.debug("extended_site: %r ", extended_site)
self.assertEquals(stuff.subsiteb_id, extended_site.parent_site._id)
self.assertEqual(2, len(extended_site.sites))
self.assertEqual(2, len(extended_site.platform_devices))
self.assertEqual(2, len(extended_site.platform_models))
self.assertIn(stuff.platform_device_id,
[o._id for o in extended_site.platform_devices])
self.assertIn(
stuff.platform_model_id,
[o._id for o in extended_site.platform_models if o is not None])
log.debug(
"verify that PlatformDeviceb is linked to PlatformDevice with hasNetworkParent link"
)
associations = self.RR.find_associations(
subject=stuff.platform_deviceb_id,
predicate=PRED.hasNetworkParent,
object=stuff.platform_device_id,
id_only=True)
self.assertIsNotNone(
associations,
"PlatformDevice child not connected to PlatformDevice parent.")
#--------------------------------------------------------------------------------
# Get the extended Org
#--------------------------------------------------------------------------------
#test the extended resource
extended_org = self.OMS.get_marine_facility_extension(stuff.org_id)
log.debug("test_observatory_org_extended: extended_org: %s ",
str(extended_org))
#self.assertEqual(2, len(extended_org.instruments_deployed) )
#self.assertEqual(1, len(extended_org.platforms_not_deployed) )
self.assertEqual(2, extended_org.number_of_platforms)
self.assertEqual(2, len(extended_org.platform_models))
self.assertEqual(2, extended_org.number_of_instruments)
self.assertEqual(2, len(extended_org.instrument_models))
self.assertEqual(1, len(extended_org.members))
self.assertNotEqual(extended_org.members[0]._id, member_actor_id)
self.assertEqual(extended_org.members[0]._id, member_user_id)
self.assertEqual(1, len(extended_org.open_requests))
self.assertTrue(len(extended_site.deployments) > 0)
self.assertEqual(len(extended_site.deployments),
len(extended_site.deployment_info))
self.assertEqual(1, extended_org.number_of_assets)
self.assertEqual(1, extended_org.number_of_asset_types)
self.assertEqual(1, extended_org.number_of_event_durations)
self.assertEqual(1, extended_org.number_of_event_duration_types)
#test the extended resource of the ION org
ion_org_id = self.org_management_service.find_org()
extended_org = self.OMS.get_marine_facility_extension(ion_org_id._id,
user_id=12345)
log.debug("test_observatory_org_extended: extended_ION_org: %s ",
str(extended_org))
self.assertEqual(1, len(extended_org.members))
self.assertEqual(0, extended_org.number_of_platforms)
#self.assertEqual(1, len(extended_org.sites))
#--------------------------------------------------------------------------------
# Get the extended Site
#--------------------------------------------------------------------------------
#create device state events to use for op /non-op filtering in extended
t = get_ion_ts()
self.event_publisher.publish_event(
ts_created=t,
event_type='ResourceAgentStateEvent',
origin=stuff.instrument_device_id,
state=ResourceAgentState.STREAMING)
self.event_publisher.publish_event(
ts_created=t,
event_type='ResourceAgentStateEvent',
origin=stuff.instrument_device2_id,
state=ResourceAgentState.INACTIVE)
extended_site = self.OMS.get_site_extension(stuff.instrument_site2_id)
log.debug("test_observatory_org_extended: extended_site: %s ",
str(extended_site))
self.dpclient.delete_data_product(data_product_id1)
class TestTransformWorker(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Instantiate a process to represent the test
process=TransformWorkerTestProcess()
self.dataset_management_client = DatasetManagementServiceClient(node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process = process)
self.time_dom, self.spatial_dom = time_series_domain()
self.ph = ParameterHelper(self.dataset_management_client, self.addCleanup)
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
def push_granule(self, data_product_id):
'''
Publishes and monitors that the granule arrived
'''
datasets, _ = self.rrclient.find_objects(data_product_id, PRED.hasDataset, id_only=True)
dataset_monitor = DatasetMonitor(datasets[0])
rdt = self.ph.rdt_for_data_product(data_product_id)
self.ph.fill_parsed_rdt(rdt)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
assert dataset_monitor.wait()
dataset_monitor.stop()
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_transform_worker(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.dp_list = []
self.data_process_objs = []
self._output_stream_ids = []
self.granule_verified = Event()
self.worker_assigned_event_verified = Event()
self.dp_created_event_verified = Event()
self.heartbeat_event_verified = Event()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream',
temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump())
self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
self.start_event_listener()
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process()
self.dp_list.append(dataprocess_id)
# validate the repository for data product algorithms persists the new resources NEW SA-1
# create_data_process call created one of each
dpd_ids, _ = self.rrclient.find_resources(restype=OT.DataProcessDefinition, id_only=False)
# there will be more than one becuase of the DPDs that reperesent the PFs in the data product above
self.assertTrue(dpd_ids is not None)
dp_ids, _ = self.rrclient.find_resources(restype=OT.DataProcess, id_only=False)
# only one DP becuase the PFs that are in the code dataproduct above are not activated yet.
self.assertEquals(len(dp_ids), 1)
# validate the name and version label NEW SA - 2
dataprocessdef_obj = self.dataprocessclient.read_data_process_definition(dataprocessdef_id)
self.assertEqual(dataprocessdef_obj.version_label, '1.0a')
self.assertEqual(dataprocessdef_obj.name, 'add_arrays')
# validate that the DPD has an attachment NEW SA - 21
attachment_ids, assoc_ids = self.rrclient.find_objects(dataprocessdef_id, PRED.hasAttachment, RT.Attachment, True)
self.assertEqual(len(attachment_ids), 1)
attachment_obj = self.rrclient.read_attachment(attachment_ids[0])
log.debug('attachment: %s', attachment_obj)
# validate that the data process resource has input and output data products associated
# L4-CI-SA-RQ-364 and NEW SA-3
outproduct_ids, assoc_ids = self.rrclient.find_objects(dataprocess_id, PRED.hasOutputProduct, RT.DataProduct, True)
self.assertEqual(len(outproduct_ids), 1)
inproduct_ids, assoc_ids = self.rrclient.find_objects(dataprocess_id, PRED.hasInputProduct, RT.DataProduct, True)
self.assertEqual(len(inproduct_ids), 1)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id,_ = self.rrclient.find_objects(subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 2)
self.assertTrue(self.input_dp_id in output_data_product_provenance[output_data_product_id[0]]['parents'])
self.assertTrue(output_data_product_provenance[output_data_product_id[0]]['parents'][self.input_dp_id]['data_process_definition_id'] == dataprocessdef_id)
# NEW SA - 4 | Data processing shall include the appropriate data product algorithm name and version number in
# the metadata of each output data product created by the data product algorithm.
output_data_product_obj,_ = self.rrclient.find_objects(subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=False)
self.assertTrue(output_data_product_obj[0].name != None)
self.assertTrue(output_data_product_obj[0]._rev != None)
# retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(subject=dataprocess_id, predicate=PRED.hasSubscription, object_type=RT.Subscription, id_only=False)
log.debug('test_transform_worker subscription_obj: %s', subscription_objs[0])
# create a queue to catch the published granules
self.subscription_id = self.pubsub_client.create_subscription(name='parsed_subscription', stream_ids=[self.stream_id], exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription, self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription, self.subscription_id)
stream_route = self.pubsub_client.read_stream_route(self.stream_id)
self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id, stream_route=stream_route )
for n in range(1, 101):
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher.publish(rdt.to_granule())
# validate that the output granule is received and the updated value is correct
self.assertTrue(self.granule_verified.wait(self.wait_time))
# validate that the data process loaded into worker event is received (L4-CI-SA-RQ-182)
self.assertTrue(self.worker_assigned_event_verified.wait(self.wait_time))
# validate that the data process create (with data product ids) event is received (NEW SA -42)
self.assertTrue(self.dp_created_event_verified.wait(self.wait_time))
# validate that the data process heartbeat event is received (for every hundred granules processed) (L4-CI-SA-RQ-182)
#this takes a while so set wait limit to large value
self.assertTrue(self.heartbeat_event_verified.wait(200))
# validate that the code from the transform function can be retrieve via inspect_data_process_definition
src = self.dataprocessclient.inspect_data_process_definition(dataprocessdef_id)
self.assertIn( 'def add_arrays(a, b)', src)
# now delete the DPD and DP then verify that the resources are retired so that information required for provenance are still available
self.dataprocessclient.delete_data_process(dataprocess_id)
self.dataprocessclient.delete_data_process_definition(dataprocessdef_id)
in_dp_objs, _ = self.rrclient.find_objects(subject=dataprocess_id, predicate=PRED.hasInputProduct, object_type=RT.DataProduct, id_only=True)
self.assertTrue(in_dp_objs is not None)
dpd_objs, _ = self.rrclient.find_subjects(subject_type=RT.DataProcessDefinition, predicate=PRED.hasDataProcess, object=dataprocess_id, id_only=True)
self.assertTrue(dpd_objs is not None)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_transform_worker_with_instrumentdevice(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# Create CTD Parsed as the initial data product
# create a stream definition for the data from the ctd simulator
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream',
temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump())
self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id)
# only ever need one device for testing purposes.
instDevice_obj,_ = self.rrclient.find_resources(restype=RT.InstrumentDevice, name='test_ctd_device')
if instDevice_obj:
instDevice_id = instDevice_obj[0]._id
else:
instDevice_obj = IonObject(RT.InstrumentDevice, name='test_ctd_device', description="test_ctd_device", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=self.input_dp_id)
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process()
self.addCleanup(self.dataprocessclient.delete_data_process, dataprocess_id)
self.addCleanup(self.dataprocessclient.delete_data_process_definition, dataprocessdef_id)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id,_ = self.rrclient.find_objects(subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 3)
self.assertTrue(self.input_dp_id in output_data_product_provenance[output_data_product_id[0]]['parents'])
self.assertTrue(instDevice_id in output_data_product_provenance[self.input_dp_id]['parents'])
self.assertTrue(output_data_product_provenance[instDevice_id]['type'] == 'InstrumentDevice')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_transform_worker_with_platformdevice(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# Create CTD Parsed as the initial data product
# create a stream definition for the data from the ctd simulator
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream',
temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump())
self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id)
# only ever need one device for testing purposes.
platform_device_obj,_ = self.rrclient.find_resources(restype=RT.PlatformDevice, name='TestPlatform')
if platform_device_obj:
platform_device_id = platform_device_obj[0]._id
else:
platform_device_obj = IonObject(RT.PlatformDevice, name='TestPlatform', description="TestPlatform", serial_number="12345" )
platform_device_id = self.imsclient.create_platform_device(platform_device=platform_device_obj)
self.damsclient.assign_data_product(input_resource_id=platform_device_id, data_product_id=self.input_dp_id)
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process()
self.addCleanup(self.dataprocessclient.delete_data_process, dataprocess_id)
self.addCleanup(self.dataprocessclient.delete_data_process_definition, dataprocessdef_id)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id,_ = self.rrclient.find_objects(subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 3)
self.assertTrue(self.input_dp_id in output_data_product_provenance[output_data_product_id[0]]['parents'])
self.assertTrue(platform_device_id in output_data_product_provenance[self.input_dp_id]['parents'])
self.assertTrue(output_data_product_provenance[platform_device_id]['type'] == 'PlatformDevice')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_event_transform_worker(self):
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# test that a data process (type: data-product-in / event-out) can be defined and launched.
# verify that event fields are correctly populated
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id)
# create the DataProduct
input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream',
temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump())
self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
# create the DPD and two DPs
self.event_data_process_id = self.create_event_data_processes()
# retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(subject=self.event_data_process_id, predicate=PRED.hasSubscription, object_type=RT.Subscription, id_only=False)
log.debug('test_event_transform_worker subscription_obj: %s', subscription_objs[0])
# create a queue to catch the published granules
self.subscription_id = self.pubsub_client.create_subscription(name='parsed_subscription', stream_ids=[self.stream_id], exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription, self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription, self.subscription_id)
stream_route = self.pubsub_client.read_stream_route(self.stream_id)
self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id, stream_route=stream_route )
self.start_event_transform_listener()
self.data_modified = Event()
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher.publish(rdt.to_granule())
self.assertTrue(self.event_verified.wait(self.wait_time))
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_bad_argument_map(self):
self._output_stream_ids = []
# test that a data process (type: data-product-in / data-product-out) parameter mapping it validated during
# data process creation and that the correct exception is raised for both input and output.
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream',
temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump())
self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# two data processes using one transform and one DPD
dp1_func_output_dp_id = self.create_output_data_product()
# Set up DPD and DP #2 - array add function
tf_obj = IonObject(RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri='http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS
)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.stream_def_id, add_array_dpd_id, binding='add_array_func' )
# create the data process with invalid argument map
argument_map = {"arr1": "foo", "arr2": "bar"}
output_param = "salinity"
with self.assertRaises(BadRequest) as cm:
dp1_data_process_id = self.dataprocessclient.create_data_process(data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id], argument_map=argument_map, out_param_name=output_param)
ex = cm.exception
log.debug(' exception raised: %s', cm)
self.assertEqual(ex.message, "Input data product does not contain the parameters defined in argument map")
# create the data process with invalid output parameter name
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "foo"
with self.assertRaises(BadRequest) as cm:
dp1_data_process_id = self.dataprocessclient.create_data_process(data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id], argument_map=argument_map, out_param_name=output_param)
ex = cm.exception
log.debug(' exception raised: %s', cm)
self.assertEqual(ex.message, "Output data product does not contain the output parameter name provided")
def create_event_data_processes(self):
# two data processes using one transform and one DPD
argument_map= {"a": "salinity"}
# set up DPD and DP #2 - array add function
tf_obj = IonObject(RT.TransformFunction,
name='validate_salinity_array',
description='validate_salinity_array',
function='validate_salinity_array',
module="ion.processes.data.transforms.test.test_transform_worker",
arguments=['a'],
function_type=TransformFunctionType.TRANSFORM
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(RT.DataProcessDefinition,
name='validate_salinity_array',
description='validate_salinity_array',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.stream_def_id, add_array_dpd_id, binding='validate_salinity_array' )
# create the data process
dp1_data_process_id = self.dataprocessclient.create_data_process(data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id],
outputs=None, argument_map=argument_map)
self.damsclient.register_process(dp1_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process, dp1_data_process_id)
return dp1_data_process_id
def create_data_process(self):
# two data processes using one transform and one DPD
dp1_func_output_dp_id = self.create_output_data_product()
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "salinity"
# set up DPD and DP #2 - array add function
tf_obj = IonObject(RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri='http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
version_label='1.0a'
)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.stream_def_id, add_array_dpd_id, binding='add_array_func' )
# create the data process
dp1_data_process_id = self.dataprocessclient.create_data_process(data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id], argument_map=argument_map, out_param_name=output_param)
self.damsclient.register_process(dp1_data_process_id)
#self.addCleanup(self.dataprocessclient.delete_data_process, dp1_data_process_id)
# add an attachment object to this DPD to test new SA-21
import msgpack
attachment_content = 'foo bar'
attachment_obj = IonObject( RT.Attachment,
name='test_attachment',
attachment_type=AttachmentType.ASCII,
content_type='text/plain',
content=msgpack.packb(attachment_content))
att_id = self.rrclient.create_attachment(add_array_dpd_id, attachment_obj)
self.addCleanup(self.rrclient.delete_attachment, att_id)
return add_array_dpd_id, dp1_data_process_id, dp1_func_output_dp_id
def create_output_data_product(self):
dp1_outgoing_stream_id = self.pubsub_client.create_stream_definition(name='dp1_stream', parameter_dictionary_id=self.parameter_dict_id)
dp1_output_dp_obj = IonObject( RT.DataProduct,
name='data_process1_data_product',
description='output of add array func',
temporal_domain = self.time_dom.dump(),
spatial_domain = self.spatial_dom.dump())
dp1_func_output_dp_id = self.dataproductclient.create_data_product(dp1_output_dp_obj, dp1_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product, dp1_func_output_dp_id)
# retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp1_func_output_dp_id, PRED.hasStream, None, True)
self._output_stream_ids.append(stream_ids[0])
subscription_id = self.pubsub_client.create_subscription('validator', data_product_ids=[dp1_func_output_dp_id])
self.addCleanup(self.pubsub_client.delete_subscription, subscription_id)
def on_granule(msg, route, stream_id):
log.debug('recv_packet stream_id: %s route: %s msg: %s', stream_id, route, msg)
self.validate_output_granule(msg, route, stream_id)
self.granule_verified.set()
validator = StandaloneStreamSubscriber('validator', callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription, subscription_id)
return dp1_func_output_dp_id
def validate_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
data_process_event = args[0]
log.debug("DataProcessStatusEvent: %s" , str(data_process_event.__dict__))
# if data process already created, check origin
if self.dp_list:
self.assertIn( data_process_event.origin, self.dp_list)
# if this is a heartbeat event then 100 granules have been processed
if 'data process status update.' in data_process_event.description:
self.heartbeat_event_verified.set()
else:
# else check that this is the assign event
if 'Data process assigned to transform worker' in data_process_event.description:
self.worker_assigned_event_verified.set()
elif 'Data process created for data product' in data_process_event.description:
self.dp_created_event_verified.set()
def validate_output_granule(self, msg, route, stream_id):
self.assertIn( stream_id, self._output_stream_ids)
rdt = RecordDictionaryTool.load_from_granule(msg)
log.debug('validate_output_granule rdt: %s', rdt)
sal_val = rdt['salinity']
np.testing.assert_array_equal(sal_val, np.array([3]))
def start_event_listener(self):
es = EventSubscriber(event_type=OT.DataProcessStatusEvent, callback=self.validate_event)
es.start()
self.addCleanup(es.stop)
def validate_transform_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
status_alert_event = args[0]
np.testing.assert_array_equal(status_alert_event.origin, self.stream_id )
np.testing.assert_array_equal(status_alert_event.values, np.array([self.event_data_process_id]))
log.debug("DeviceStatusAlertEvent: %s" , str(status_alert_event.__dict__))
self.event_verified.set()
def start_event_transform_listener(self):
es = EventSubscriber(event_type=OT.DeviceStatusAlertEvent, callback=self.validate_transform_event)
es.start()
self.addCleanup(es.stop)
def test_download(self):
egg_url = 'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
egg_path = TransformWorker.download_egg(egg_url)
import pkg_resources
pkg_resources.working_set.add_entry(egg_path)
from ion_example.add_arrays import add_arrays
a = add_arrays(1,2)
self.assertEquals(a,3)
class TransformManagementServiceIntTest(IonIntegrationTestCase):
def setUp(self):
# set up the container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
self.tms_cli = TransformManagementServiceClient(node=self.container.node)
self.rr_cli = ResourceRegistryServiceClient(node=self.container.node)
self.procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.input_stream_id = self.pubsub_cli.create_stream(name='input_stream',original=True)
self.input_subscription_id = self.pubsub_cli.create_subscription(query=StreamQuery(stream_ids=[self.input_stream_id]),exchange_name='transform_input',name='input_subscription')
self.output_stream_id = self.pubsub_cli.create_stream(name='output_stream',original=True)
self.process_definition = ProcessDefinition(name='basic_transform_definition')
self.process_definition.executable = {'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'}
self.process_definition_id = self.procd_cli.create_process_definition(process_definition=self.process_definition)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
def test_create_transform(self):
configuration = {'program_args':{'arg1':'value'}}
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id)
# test transform creation in rr
transform = self.rr_cli.read(transform_id)
self.assertEquals(transform.name,'test_transform')
# test associations
predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition]
assocs = []
for p in predicates:
assocs += self.rr_cli.find_associations(transform_id,p,id_only=True)
self.assertEquals(len(assocs),3)
# test process creation
transform = self.tms_cli.read_transform(transform_id)
pid = transform.process_id
proc = self.container.proc_manager.procs.get(pid)
self.assertIsInstance(proc,TransformExample)
# clean up
self.tms_cli.delete_transform(transform_id)
def test_create_transform_no_procdef(self):
with self.assertRaises(NotFound):
self.tms_cli.create_transform(name='test',in_subscription_id=self.input_subscription_id)
def test_create_transform_bad_procdef(self):
with self.assertRaises(NotFound):
self.tms_cli.create_transform(name='test',
in_subscription_id=self.input_subscription_id,
process_definition_id='bad')
def test_create_transform_no_config(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
self.tms_cli.delete_transform(transform_id)
def test_create_transform_name_failure(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
with self.assertRaises(BadRequest):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
self.tms_cli.delete_transform(transform_id)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
def test_create_no_output(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
process_definition_id=self.process_definition_id,
)
predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition]
assocs = []
for p in predicates:
assocs += self.rr_cli.find_associations(transform_id,p,id_only=True)
self.assertEquals(len(assocs),2)
# test process creation
transform = self.tms_cli.read_transform(transform_id)
pid = transform.process_id
proc = self.container.proc_manager.procs.get(pid)
self.assertIsInstance(proc,TransformExample)
self.tms_cli.delete_transform(transform_id)
def test_read_transform_exists(self):
trans_obj = IonObject(RT.Transform,name='trans_obj')
trans_id, _ = self.rr_cli.create(trans_obj)
res = self.tms_cli.read_transform(trans_id)
actual = self.rr_cli.read(trans_id)
self.assertEquals(res._id,actual._id)
def test_read_transform_nonexist(self):
with self.assertRaises(NotFound) as e:
self.tms_cli.read_transform('123')
def test_activate_transform(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id
)
self.tms_cli.activate_transform(transform_id)
# pubsub check if activated?
self.tms_cli.delete_transform(transform_id)
def test_activate_transform_nonexist(self):
with self.assertRaises(NotFound):
self.tms_cli.activate_transform('1234')
def test_delete_transform(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
process_definition_id=self.process_definition_id
)
self.tms_cli.delete_transform(transform_id)
# assertions
with self.assertRaises(NotFound):
self.rr_cli.read(transform_id)
def test_delete_transform_nonexist(self):
with self.assertRaises(NotFound):
self.tms_cli.delete_transform('123')
def test_execute_transform(self):
# set up
process_definition = ProcessDefinition(name='procdef_execute')
process_definition.executable['module'] = 'ion.processes.data.transforms.transform_example'
process_definition.executable['class'] = 'ReverseTransform'
data = [1,2,3]
process_definition_id, _ = self.rr_cli.create(process_definition)
retval = self.tms_cli.execute_transform(process_definition_id,data)
self.assertEquals(retval,[3,2,1])
def test_integrated_transform(self):
'''
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
'''
cc = self.container
assertions = self.assertTrue
pubsub_cli = PubsubManagementServiceClient(node=cc.node)
rr_cli = ResourceRegistryServiceClient(node=cc.node)
tms_cli = TransformManagementServiceClient(node=cc.node)
#-------------------------------
# Process Definition
#-------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'
}
basic_transform_definition_id, _ = rr_cli.create(process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformEvenOdd'
}
evenodd_transform_definition_id, _ = rr_cli.create(process_definition)
#-------------------------------
# Streams
#-------------------------------
streams = [pubsub_cli.create_stream() for i in xrange(5)]
#-------------------------------
# Subscriptions
#-------------------------------
query = StreamQuery(stream_ids=[streams[0]])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
query = StreamQuery(stream_ids = [streams[1]]) # even output
even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue')
query = StreamQuery(stream_ids = [streams[2]]) # odd output
odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue')
#-------------------------------
# Launch the EvenOdd Transform
#-------------------------------
evenodd_id = tms_cli.create_transform(name='even_odd',
in_subscription_id=input_subscription_id,
out_streams={'even':streams[1], 'odd':streams[2]},
process_definition_id=evenodd_transform_definition_id,
configuration={})
tms_cli.activate_transform(evenodd_id)
#-------------------------------
# Launch the Even Processing Transform
#-------------------------------
even_transform_id = tms_cli.create_transform(name='even_transform',
in_subscription_id = even_subscription_id,
out_streams={'even_plus1':streams[3]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(even_transform_id)
#-------------------------------
# Launch the Odd Processing Transform
#-------------------------------
odd_transform_id = tms_cli.create_transform(name='odd_transform',
in_subscription_id = odd_subscription_id,
out_streams={'odd_plus1':streams[4]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(odd_transform_id)
#-------------------------------
# Set up final subscribers
#-------------------------------
evenplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[3]]),
exchange_name='evenplus1_queue',
name='EvenPlus1Subscription',
description='EvenPlus1 SubscriptionDescription'
)
oddplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[4]]),
exchange_name='oddplus1_queue',
name='OddPlus1Subscription',
description='OddPlus1 SubscriptionDescription'
)
total_msg_count = 2
msgs = gevent.queue.Queue()
def even1_message_received(message, headers):
input = int(message.get('num'))
assertions( (input % 2) ) # Assert it is odd (transform adds 1)
msgs.put(True)
def odd1_message_received(message, headers):
input = int(message.get('num'))
assertions(not (input % 2)) # Assert it is even
msgs.put(True)
subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node)
even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received)
odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received)
# Start subscribers
even_subscriber.start()
odd_subscriber.start()
# Activate subscriptions
pubsub_cli.activate_subscription(evenplus1_subscription_id)
pubsub_cli.activate_subscription(oddplus1_subscription_id)
#-------------------------------
# Set up fake stream producer
#-------------------------------
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node)
stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0])
#-------------------------------
# Start test
#-------------------------------
# Publish a stream
for i in xrange(total_msg_count):
stream_publisher.publish({'num':str(i)})
time.sleep(0.5)
for i in xrange(total_msg_count * 2):
try:
msgs.get()
except Empty:
assertions(False, "Failed to process all messages correctly.")
"""
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.identcli = IdentityManagementServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
@attr('EXT')
@attr('PREP')
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
# Assert that the data product has an associated stream at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp')
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
group_names = self.dpsc_cli.get_data_product_group_list()
self.assertIn("PRODNAME", group_names)
#----------------------------------------------------------------------------------------
# Create users then notifications to this data product for each user
#----------------------------------------------------------------------------------------
# user_1
user_1 = UserInfo()
user_1.name = 'user_1'
user_1.contact.email = '*****@*****.**'
# user_2
user_2 = UserInfo()
user_2.name = 'user_2'
user_2.contact.email = '*****@*****.**'
#user1 is a complete user
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
actor_id = self.identcli.create_actor_identity(actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials", {"name": self.subject})
self.identcli.register_user_credentials(actor_id, user_credentials_obj)
user_id_1 = self.identcli.create_user_info(actor_id, user_1)
user_id_2, _ = self.rrclient.create(user_2)
delivery_config1a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
delivery_config1b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
notification_request_1 = NotificationRequest( name = "notification_1",
origin=dp_id,
origin_type="type_1",
event_type=OT.ResourceLifecycleEvent,
disabled_by_system = False,
delivery_configurations=[delivery_config1a, delivery_config1b])
delivery_config2a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
delivery_config2b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
notification_request_2 = NotificationRequest( name = "notification_2",
origin=dp_id,
origin_type="type_2",
disabled_by_system = False,
event_type=OT.DetectionEvent,
delivery_configurations=[delivery_config2a, delivery_config2b])
notification_request_1_id = self.unsc.create_notification(notification=notification_request_1, user_id=user_id_1)
notification_request_2_id = self.unsc.create_notification(notification=notification_request_2, user_id=user_id_2)
self.unsc.delete_notification(notification_request_1_id)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description,'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
#validate that there is one active and one retired user notification for this data product
self.assertEqual(1, len(extended_product.computed.active_user_subscriptions.value))
self.assertEqual(1, len(extended_product.computed.past_user_subscriptions.value))
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
def ion_object_encoder(obj):
return obj.__dict__
#test prepare for create
data_product_data = self.dpsc_cli.prepare_data_product_support()
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, "")
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 0)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0)
#test prepare for update
data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, dp_id)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 1)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1)
self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, dp_id)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 1)
self.assertEqual(data_product_data.associations['Dataset'].associated_resources[0].s, dp_id)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
# Assert that there are no associated streams leftover after deleting the data product
stream_ids, assoc_ids = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertEquals(len(stream_ids), 0)
self.assertEquals(len(assoc_ids), 0)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_derived_data_product(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id)
dp = DataProduct(name='Instrument DP')
dp_id = self.dpsc_cli.create_data_product(dp, stream_definition_id=ctd_stream_def_id)
self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Make the derived data product
simple_stream_def_id = self.pubsubcli.create_stream_definition(name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time','temp'])
tempwat_dp = DataProduct(name='TEMPWAT', category=DataProductTypeEnum.DERIVED)
tempwat_dp_id = self.dpsc_cli.create_data_product(tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id)
self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id)
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
rdt['pressure'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True)
tempwat_dataset_id = tempwat_dataset_ids[0]
granule = self.data_retriever.retrieve(tempwat_dataset_id, delivery_format=simple_stream_def_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
self.assertEquals(set(rdt.fields), set(['time','temp']))
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# Subscribe to persist events
#------------------------------------------------------------------------------------------------
queue = gevent.queue.Queue()
def info_event_received(message, headers):
queue.put(message)
es = EventSubscriber(event_type=OT.InformationContentStatusEvent, callback=info_event_received, origin=dp_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug("The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'")
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name,'DP1')
self.assertEquals(data_product_object.description,'some new dp')
log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
dataset_modified.clear()
rdt['time'] = np.arange(20,40)
publisher.publish(rdt.to_granule())
self.assertFalse(dataset_modified.wait(2))
self.dpsc_cli.activate_data_product_persistence(dp_id)
dataset_modified.clear()
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt['time'], np.arange(40))
dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True)
self.assertEquals(len(dataset_ids), 1)
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
info_event_counter = 0
runtime = 0
starttime = time.time()
caught_events = []
#check that the four InfoStatusEvents were received
while info_event_counter < 4 and runtime < 60 :
a = queue.get(timeout=60)
caught_events.append(a)
info_event_counter += 1
runtime = time.time() - starttime
self.assertEquals(info_event_counter, 4)
def test_integrated_transform(self):
'''
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
'''
cc = self.container
assertions = self.assertTrue
pubsub_cli = PubsubManagementServiceClient(node=cc.node)
rr_cli = ResourceRegistryServiceClient(node=cc.node)
tms_cli = TransformManagementServiceClient(node=cc.node)
#-------------------------------
# Process Definition
#-------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'
}
basic_transform_definition_id, _ = rr_cli.create(process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformEvenOdd'
}
evenodd_transform_definition_id, _ = rr_cli.create(process_definition)
#-------------------------------
# Streams
#-------------------------------
streams = [pubsub_cli.create_stream() for i in xrange(5)]
#-------------------------------
# Subscriptions
#-------------------------------
query = StreamQuery(stream_ids=[streams[0]])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
query = StreamQuery(stream_ids = [streams[1]]) # even output
even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue')
query = StreamQuery(stream_ids = [streams[2]]) # odd output
odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue')
#-------------------------------
# Launch the EvenOdd Transform
#-------------------------------
evenodd_id = tms_cli.create_transform(name='even_odd',
in_subscription_id=input_subscription_id,
out_streams={'even':streams[1], 'odd':streams[2]},
process_definition_id=evenodd_transform_definition_id,
configuration={})
tms_cli.activate_transform(evenodd_id)
#-------------------------------
# Launch the Even Processing Transform
#-------------------------------
even_transform_id = tms_cli.create_transform(name='even_transform',
in_subscription_id = even_subscription_id,
out_streams={'even_plus1':streams[3]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(even_transform_id)
#-------------------------------
# Launch the Odd Processing Transform
#-------------------------------
odd_transform_id = tms_cli.create_transform(name='odd_transform',
in_subscription_id = odd_subscription_id,
out_streams={'odd_plus1':streams[4]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(odd_transform_id)
#-------------------------------
# Set up final subscribers
#-------------------------------
evenplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[3]]),
exchange_name='evenplus1_queue',
name='EvenPlus1Subscription',
description='EvenPlus1 SubscriptionDescription'
)
oddplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[4]]),
exchange_name='oddplus1_queue',
name='OddPlus1Subscription',
description='OddPlus1 SubscriptionDescription'
)
total_msg_count = 2
msgs = gevent.queue.Queue()
def even1_message_received(message, headers):
input = int(message.get('num'))
assertions( (input % 2) ) # Assert it is odd (transform adds 1)
msgs.put(True)
def odd1_message_received(message, headers):
input = int(message.get('num'))
assertions(not (input % 2)) # Assert it is even
msgs.put(True)
subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node)
even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received)
odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received)
# Start subscribers
even_subscriber.start()
odd_subscriber.start()
# Activate subscriptions
pubsub_cli.activate_subscription(evenplus1_subscription_id)
pubsub_cli.activate_subscription(oddplus1_subscription_id)
#-------------------------------
# Set up fake stream producer
#-------------------------------
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node)
stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0])
#-------------------------------
# Start test
#-------------------------------
# Publish a stream
for i in xrange(total_msg_count):
stream_publisher.publish({'num':str(i)})
time.sleep(0.5)
for i in xrange(total_msg_count * 2):
try:
msgs.get()
except Empty:
assertions(False, "Failed to process all messages correctly.")
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
print 'started services'
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
sensor_model_id #is only a target
#create a parsed product for this instrument output
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain = tdom,
spatial_domain = sdom)
pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
log.debug("extended_instrument.computed: %s", extended_instrument.computed)
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name)
#check agent instance
inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id)
self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
#compound assoc return list of lists so check the first element
self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent[0].name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name)
extended_platform = self.IMS.get_platform_device_extension(platform_device_id)
self.assertEqual(1, len(extended_platform.instrument_devices))
self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id)
self.assertEqual(1, len(extended_platform.instrument_models))
self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id)
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
#check data_product_parameters_set
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.data_product_parameters_set.status)
self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value)
# the ctd parameters should include 'temp'
self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical'])
#none of these will work because there is no agent
self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
extended_instrument.computed.firmware_version.status)
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.operational_state.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.power_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.communications_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.data_status_roll_up.status)
# self.assertEqual(StatusType.STATUS_OK,
# extended_instrument.computed.data_status_roll_up.value)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.location_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.recent_events.status)
# self.assertEqual([], extended_instrument.computed.recent_events.value)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
def test_custom_attributes(self):
"""
Test assignment of custom attributes
"""
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel,
{"custom_attributes":
{"favorite_color": "attr desc goes here"}
}))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice,
{"custom_attributes":
{"favorite_color": "red",
"bogus_attr": "should raise warning"
}
}))
self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
# cleanup
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
def _get_datastore(self, dataset_id):
dataset = self.DSC.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_resource_state_save_restore(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg",
stream_configurations = [raw_config, parsed_config] )
instAgent_id = self.IMS.create_instrument_agent(instAgent_obj)
log.debug( 'new InstrumentAgent id = %s', instAgent_id)
self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 '
+ '(SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
log.debug( 'new dp_id = %s', str(data_product_id2))
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
# spin up agent
self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.IMS.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
instance_obj.agent_process_id)
# take snapshot of config
snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot")
snap_obj = self.RR.read_attachment(snap_id, include_content=True)
print "Saved config:"
print snap_obj.content
#modify config
instance_obj.driver_config["comms_config"] = "BAD_DATA"
self.RR.update(instance_obj)
#restore config
self.IMS.restore_resource_state(instDevice_id, snap_id)
instance_obj = self.RR.read(instAgentInstance_id)
self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"])
self.DP.delete_data_product(data_product_id1)
self.DP.delete_data_product(data_product_id2)
def test_agent_instance_config(self):
"""
Verify that agent configurations are being built properly
"""
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_id = self.RR2.create(any_old(RT.Org))
inst_startup_config = {'startup': 'config'}
required_config_keys = [
'org_name',
'device_type',
'agent',
'driver_config',
'stream_config',
'startup_config',
'alarm_defs',
'children']
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'children']:
self.assertEqual({}, config[key])
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config'])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
if None is inst_device_id:
for key in ['alarm_defs', 'children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id], inst_device_id)
def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance)
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)
log.debug("Making the structure for a platform agent, which will be the child")
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure()
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj)
child_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug("Making the structure for a platform agent, which will be the parent")
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure()
platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure()
instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
full_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)
#self.fail(parent_config)
#plauncher.prepare(will_launch=False)
def sample_nested_platform_agent_instance_config(self):
"""
for informational purposes
"""
ret = {'org_name': 'Org_1',
'alarm_defs': {},
'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)},
'stream_config': {'parameter_dictionary': 'lots of stuff'},
'agent': {'resource_id': '33e54106c4444444862da082098bc123'},
'startup_config': {},
'device_type': 'PlatformDevice',
'children': {'76a39596eeff4fd5b409c4cb93f0e581':
{'org_name': 'Org_1',
'alarm_defs': {},
'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)},
'stream_config': {'parameter_dictionary': 'lots of stuff'},
'agent': {'resource_id': '76a39596eeff4fd5b409c4cb93f0e581'},
'startup_config': {},
'device_type': 'PlatformDevice',
'children': {}}}}
return ret
class IngestionManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
self.ingestion_management = IngestionManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.ingest_name = "basic"
self.exchange = "testdata"
@staticmethod
def clean_subscriptions():
ingestion_management = IngestionManagementServiceClient()
pubsub = PubsubManagementServiceClient()
rr = ResourceRegistryServiceClient()
ingestion_config_ids = ingestion_management.list_ingestion_configurations(id_only=True)
for ic in ingestion_config_ids:
assocs = rr.find_associations(subject=ic, predicate=PRED.hasSubscription, id_only=False)
for assoc in assocs:
rr.delete_association(assoc)
try:
pubsub.deactivate_subscription(assoc.o)
except:
pass
pubsub.delete_subscription(assoc.o)
def create_ingest_config(self):
self.queue = IngestionQueue(name="test", type="testdata")
# Create the ingestion config
ingestion_config_id = self.ingestion_management.create_ingestion_configuration(
name=self.ingest_name, exchange_point_id=self.exchange, queues=[self.queue]
)
return ingestion_config_id
def test_ingestion_config_crud(self):
ingestion_config_id = self.create_ingest_config()
ingestion_config = self.ingestion_management.read_ingestion_configuration(ingestion_config_id)
self.assertTrue(ingestion_config.name == self.ingest_name)
self.assertTrue(ingestion_config.queues[0].name == "test")
self.assertTrue(ingestion_config.queues[0].type == "testdata")
ingestion_config.name = "another"
self.ingestion_management.update_ingestion_configuration(ingestion_config)
# Create an association just to make sure that it will delete them
sub = Subscription()
sub_id, _ = self.resource_registry.create(sub)
assoc_id, _ = self.resource_registry.create_association(
subject=ingestion_config_id, predicate=PRED.hasSubscription, object=sub_id
)
self.ingestion_management.delete_ingestion_configuration(ingestion_config_id)
with self.assertRaises(NotFound):
self.resource_registry.read(assoc_id)
def test_list_ingestion(self):
# Create the ingest_config
config_id = self.create_ingest_config()
retval = self.ingestion_management.list_ingestion_configurations(id_only=True)
# Nice thing about this is that it breaks if r2dm adds an ingest_config
self.assertTrue(config_id in retval)
class TestObservatoryManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.org_management_service = OrgManagementServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#print 'TestObservatoryManagementServiceIntegration: started services'
self.event_publisher = EventPublisher()
# @unittest.skip('this exists only for debugging the launch process')
# def test_just_the_setup(self):
# return
def destroy(self, resource_ids):
self.OMS.force_delete_observatory(resource_ids.observatory_id)
self.OMS.force_delete_subsite(resource_ids.subsite_id)
self.OMS.force_delete_subsite(resource_ids.subsite2_id)
self.OMS.force_delete_subsite(resource_ids.subsiteb_id)
self.OMS.force_delete_subsite(resource_ids.subsitez_id)
self.OMS.force_delete_platform_site(resource_ids.platform_site_id)
self.OMS.force_delete_platform_site(resource_ids.platform_siteb_id)
self.OMS.force_delete_platform_site(resource_ids.platform_siteb2_id)
self.OMS.force_delete_platform_site(resource_ids.platform_site3_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_site_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_site2_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_siteb3_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_site4_id)
#@unittest.skip('targeting')
def test_resources_associations(self):
resources = self._make_associations()
self.destroy(resources)
#@unittest.skip('targeting')
def test_find_related_frames_of_reference(self):
# finding subordinates gives a dict of obj lists, convert objs to ids
def idify(adict):
ids = {}
for k, v in adict.iteritems():
ids[k] = []
for obj in v:
ids[k].append(obj._id)
return ids
# a short version of the function we're testing, with id-ify
def short(resource_id, output_types):
ret = self.OMS.find_related_frames_of_reference(resource_id,
output_types)
return idify(ret)
#set up associations first
stuff = self._make_associations()
#basic traversal of tree from instrument to platform
ids = short(stuff.instrument_site_id, [RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
#since this is the first search, just make sure the input inst_id got stripped
if RT.InstrumentSite in ids:
self.assertNotIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
#basic traversal of tree from platform to instrument
ids = short(stuff.platform_siteb_id, [RT.InstrumentSite])
self.assertIn(RT.InstrumentSite, ids)
self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
self.assertNotIn(stuff.instrument_site2_id, ids[RT.InstrumentSite])
#full traversal of tree from observatory down to instrument
ids = short(stuff.observatory_id, [RT.InstrumentSite])
self.assertIn(RT.InstrumentSite, ids)
self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
#full traversal of tree from instrument to observatory
ids = short(stuff.instrument_site_id, [RT.Observatory])
self.assertIn(RT.Observatory, ids)
self.assertIn(stuff.observatory_id, ids[RT.Observatory])
#partial traversal, only down to platform
ids = short(stuff.observatory_id, [RT.Subsite, RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(RT.Subsite, ids)
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_site3_id, ids[RT.PlatformSite])
self.assertIn(stuff.subsite_id, ids[RT.Subsite])
self.assertIn(stuff.subsite2_id, ids[RT.Subsite])
self.assertIn(stuff.subsitez_id, ids[RT.Subsite])
self.assertIn(stuff.subsiteb_id, ids[RT.Subsite])
self.assertNotIn(RT.InstrumentSite, ids)
#partial traversal, only down to platform
ids = short(stuff.instrument_site_id, [RT.Subsite, RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(RT.Subsite, ids)
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.subsite_id, ids[RT.Subsite])
self.assertIn(stuff.subsiteb_id, ids[RT.Subsite])
self.assertNotIn(stuff.subsite2_id, ids[RT.Subsite])
self.assertNotIn(stuff.subsitez_id, ids[RT.Subsite])
self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
self.assertNotIn(RT.Observatory, ids)
self.destroy(stuff)
def _make_associations(self):
"""
create one of each resource and association used by OMS
to guard against problems in ion-definitions
"""
#raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41")
"""
the tree we're creating (observatory, sites, platforms, instruments)
rows are lettered, colums numbered.
- first row is implied a
- first column is implied 1
- site Z, just because
O--Sz
|
S--S2--P3--I4
|
Sb-Pb2-Ib3
|
P--I2 <- PlatformDevice, InstrumentDevice2
|
Pb <- PlatformDevice b
|
I <- InstrumentDevice
"""
org_id = self.OMS.create_marine_facility(any_old(RT.Org))
def create_under_org(resource_type):
obj = any_old(resource_type)
if RT.InstrumentDevice == resource_type:
resource_id = self.IMS.create_instrument_device(obj)
else:
resource_id, _ = self.RR.create(obj)
self.OMS.assign_resource_to_observatory_org(resource_id=resource_id, org_id=org_id)
return resource_id
#stuff we control
observatory_id = create_under_org(RT.Observatory)
subsite_id = create_under_org(RT.Subsite)
subsite2_id = create_under_org(RT.Subsite)
subsiteb_id = create_under_org(RT.Subsite)
subsitez_id = create_under_org(RT.Subsite)
platform_site_id = create_under_org(RT.PlatformSite)
platform_siteb_id = create_under_org(RT.PlatformSite)
platform_siteb2_id = create_under_org(RT.PlatformSite)
platform_site3_id = create_under_org(RT.PlatformSite)
instrument_site_id = create_under_org(RT.InstrumentSite)
instrument_site2_id = create_under_org(RT.InstrumentSite)
instrument_siteb3_id = create_under_org(RT.InstrumentSite)
instrument_site4_id = create_under_org(RT.InstrumentSite)
#stuff we associate to
instrument_device_id = create_under_org(RT.InstrumentDevice)
instrument_device2_id = create_under_org(RT.InstrumentDevice)
platform_device_id = create_under_org(RT.PlatformDevice)
platform_deviceb_id = create_under_org(RT.PlatformDevice)
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
deployment_id, _ = self.RR.create(any_old(RT.Deployment))
#observatory
self.RR.create_association(observatory_id, PRED.hasSite, subsite_id)
self.RR.create_association(observatory_id, PRED.hasSite, subsitez_id)
#site
self.RR.create_association(subsite_id, PRED.hasSite, subsite2_id)
self.RR.create_association(subsite_id, PRED.hasSite, subsiteb_id)
self.RR.create_association(subsite2_id, PRED.hasSite, platform_site3_id)
self.RR.create_association(subsiteb_id, PRED.hasSite, platform_siteb2_id)
self.RR.create_association(subsiteb_id, PRED.hasSite, platform_site_id)
#platform_site(s)
self.RR.create_association(platform_site3_id, PRED.hasSite, instrument_site4_id)
self.RR.create_association(platform_siteb2_id, PRED.hasSite, instrument_siteb3_id)
self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site2_id)
self.RR.create_association(platform_site_id, PRED.hasSite, platform_siteb_id)
self.RR.create_association(platform_siteb_id, PRED.hasSite, instrument_site_id)
self.RR.create_association(platform_siteb_id, PRED.hasDevice, platform_deviceb_id)
self.RR.create_association(platform_site_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id)
self.RR.create_association(platform_site_id, PRED.hasDeployment, deployment_id)
#instrument_site(s)
self.RR.create_association(instrument_site_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id)
self.RR.create_association(instrument_site_id, PRED.hasDeployment, deployment_id)
self.RR.create_association(instrument_site2_id, PRED.hasDevice, instrument_device2_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id)
ret = DotDict()
ret.org_id = org_id
ret.observatory_id = observatory_id
ret.subsite_id = subsite_id
ret.subsite2_id = subsite2_id
ret.subsiteb_id = subsiteb_id
ret.subsitez_id = subsitez_id
ret.platform_site_id = platform_site_id
ret.platform_siteb_id = platform_siteb_id
ret.platform_siteb2_id = platform_siteb2_id
ret.platform_site3_id = platform_site3_id
ret.instrument_site_id = instrument_site_id
ret.instrument_site2_id = instrument_site2_id
ret.instrument_siteb3_id = instrument_siteb3_id
ret.instrument_site4_id = instrument_site4_id
ret.instrument_device_id = instrument_device_id
ret.instrument_device2_id = instrument_device2_id
ret.platform_device_id = platform_device_id
ret.platform_deviceb_id = platform_deviceb_id
ret.instrument_model_id = instrument_model_id
ret.platform_model_id = platform_model_id
ret.deployment_id = deployment_id
return ret
#@unittest.skip("targeting")
def test_create_observatory(self):
observatory_obj = IonObject(RT.Observatory,
name='TestFacility',
description='some new mf')
observatory_id = self.OMS.create_observatory(observatory_obj)
self.OMS.force_delete_observatory(observatory_id)
#@unittest.skip("targeting")
def test_find_observatory_org(self):
org_obj = IonObject(RT.Org,
name='TestOrg',
description='some new mf org')
org_id = self.OMS.create_marine_facility(org_obj)
observatory_obj = IonObject(RT.Observatory,
name='TestObservatory',
description='some new obs')
observatory_id = self.OMS.create_observatory(observatory_obj)
#make association
self.OMS.assign_resource_to_observatory_org(observatory_id, org_id)
#find association
org_objs = self.OMS.find_org_by_observatory(observatory_id)
self.assertEqual(1, len(org_objs))
self.assertEqual(org_id, org_objs[0]._id)
print("org_id=<" + org_id + ">")
#create a subsite with parent Observatory
subsite_obj = IonObject(RT.Subsite,
name= 'TestSubsite',
description = 'sample subsite')
subsite_id = self.OMS.create_subsite(subsite_obj, observatory_id)
self.assertIsNotNone(subsite_id, "Subsite not created.")
# verify that Subsite is linked to Observatory
mf_subsite_assoc = self.RR.get_association(observatory_id, PRED.hasSite, subsite_id)
self.assertIsNotNone(mf_subsite_assoc, "Subsite not connected to Observatory.")
# add the Subsite as a resource of this Observatory
self.OMS.assign_resource_to_observatory_org(resource_id=subsite_id, org_id=org_id)
# verify that Subsite is linked to Org
org_subsite_assoc = self.RR.get_association(org_id, PRED.hasResource, subsite_id)
self.assertIsNotNone(org_subsite_assoc, "Subsite not connected as resource to Org.")
#create a logical platform with parent Subsite
platform_site_obj = IonObject(RT.PlatformSite,
name= 'TestPlatformSite',
description = 'sample logical platform')
platform_site_id = self.OMS.create_platform_site(platform_site_obj, subsite_id)
self.assertIsNotNone(platform_site_id, "PlatformSite not created.")
# verify that PlatformSite is linked to Site
site_lp_assoc = self.RR.get_association(subsite_id, PRED.hasSite, platform_site_id)
self.assertIsNotNone(site_lp_assoc, "PlatformSite not connected to Site.")
# add the PlatformSite as a resource of this Observatory
self.OMS.assign_resource_to_observatory_org(resource_id=platform_site_id, org_id=org_id)
# verify that PlatformSite is linked to Org
org_lp_assoc = self.RR.get_association(org_id, PRED.hasResource, platform_site_id)
self.assertIsNotNone(org_lp_assoc, "PlatformSite not connected as resource to Org.")
#create a logical instrument with parent logical platform
instrument_site_obj = IonObject(RT.InstrumentSite,
name= 'TestInstrumentSite',
description = 'sample logical instrument')
instrument_site_id = self.OMS.create_instrument_site(instrument_site_obj, platform_site_id)
self.assertIsNotNone(instrument_site_id, "InstrumentSite not created.")
# verify that InstrumentSite is linked to PlatformSite
li_lp_assoc = self.RR.get_association(platform_site_id, PRED.hasSite, instrument_site_id)
self.assertIsNotNone(li_lp_assoc, "InstrumentSite not connected to PlatformSite.")
# add the InstrumentSite as a resource of this Observatory
self.OMS.assign_resource_to_observatory_org(resource_id=instrument_site_id, org_id=org_id)
# verify that InstrumentSite is linked to Org
org_li_assoc = self.RR.get_association(org_id, PRED.hasResource, instrument_site_id)
self.assertIsNotNone(org_li_assoc, "InstrumentSite not connected as resource to Org.")
# remove the InstrumentSite as a resource of this Observatory
self.OMS.unassign_resource_from_observatory_org(instrument_site_id, org_id)
# verify that InstrumentSite is linked to Org
assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.InstrumentSite, id_only=True )
self.assertEqual(len(assocs), 0)
# remove the InstrumentSite
self.OMS.delete_instrument_site(instrument_site_id)
assocs, _ = self.RR.find_objects(platform_site_id, PRED.hasSite, RT.InstrumentSite, id_only=True )
self.assertEqual(len(assocs), 1)
#todo: remove the dangling association
# remove the PlatformSite as a resource of this Observatory
self.OMS.unassign_resource_from_observatory_org(platform_site_id, org_id)
# verify that PlatformSite is linked to Org
assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.PlatformSite, id_only=True )
self.assertEqual(len(assocs), 0)
# remove the Site as a resource of this Observatory
self.OMS.unassign_resource_from_observatory_org(subsite_id, org_id)
# verify that Site is linked to Org
assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.Subsite, id_only=True )
self.assertEqual(len(assocs), 0)
self.RR.delete(org_id)
self.OMS.force_delete_observatory(observatory_id)
self.OMS.force_delete_subsite(subsite_id)
self.OMS.force_delete_platform_site(platform_site_id)
self.OMS.force_delete_instrument_site(instrument_site_id)
#@unittest.skip("in development...")
@attr('EXT')
def test_observatory_org_extended(self):
stuff = self._make_associations()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',
id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed',
parameter_dictionary_id=parsed_pdict_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=stuff.instrument_device_id,
data_product_id=data_product_id1)
#--------------------------------------------------------------------------------
# Get the extended Site (platformSite)
#--------------------------------------------------------------------------------
extended_site = self.OMS.get_site_extension(stuff.platform_site_id)
log.debug("extended_site: %s ", str(extended_site))
self.assertEqual(1, len(extended_site.platform_devices))
self.assertEqual(1, len(extended_site.platform_models))
self.assertEqual(stuff.platform_device_id, extended_site.platform_devices[0]._id)
self.assertEqual(stuff.platform_model_id, extended_site.platform_models[0]._id)
#--------------------------------------------------------------------------------
# Get the extended Org
#--------------------------------------------------------------------------------
#test the extended resource
extended_org = self.org_management_service.get_marine_facility_extension(stuff.org_id)
log.debug("test_observatory_org_extended: extended_org: %s ", str(extended_org))
#self.assertEqual(2, len(extended_org.instruments_deployed) )
#self.assertEqual(1, len(extended_org.platforms_not_deployed) )
self.assertEqual(2, extended_org.number_of_platforms)
self.assertEqual(2, len(extended_org.platform_models) )
self.assertEqual(2, extended_org.number_of_instruments)
self.assertEqual(2, len(extended_org.instrument_models) )
#test the extended resource of the ION org
ion_org_id = self.org_management_service.find_org()
extended_org = self.org_management_service.get_marine_facility_extension(ion_org_id._id)
log.debug("test_observatory_org_extended: extended_ION_org: %s ", str(extended_org))
self.assertEqual(0, len(extended_org.members))
self.assertEqual(0, extended_org.number_of_platforms)
#self.assertEqual(1, len(extended_org.sites))
#--------------------------------------------------------------------------------
# Get the extended Site
#--------------------------------------------------------------------------------
#create device state events to use for op /non-op filtering in extended
t = get_ion_ts()
self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent',
origin = stuff.instrument_device_id, state=ResourceAgentState.STREAMING )
self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent',
origin = stuff.instrument_device2_id, state=ResourceAgentState.INACTIVE )
extended_site = self.OMS.get_site_extension(stuff.instrument_site2_id)
log.debug("test_observatory_org_extended: extended_site: %s ", str(extended_site))
self.dpclient.delete_data_product(data_product_id1)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
datastore_name = CACHE_DATASTORE_NAME
self.db = self.container.datastore_manager.get_datastore(datastore_name)
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
@attr('EXT')
@attr('PREP')
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Generic time-series data domain creation
tdom, sdom = time_series_domain()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
# Assert that the data product has an associated stream at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
group_names = self.dpsc_cli.get_data_product_group_list()
self.assertIn("PRODNAME", group_names)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description,'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
def ion_object_encoder(obj):
return obj.__dict__
#test prepare for create
data_product_data = self.dpsc_cli.prepare_data_product_support()
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, "")
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 0)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0)
#test prepare for update
data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, dp_id)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 1)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1)
self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, dp_id)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 1)
self.assertEqual(data_product_data.associations['Dataset'].associated_resources[0].s, dp_id)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
# Assert that there are no associated streams leftover after deleting the data product
stream_ids, assoc_ids = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertEquals(len(stream_ids), 0)
self.assertEquals(len(assoc_ids), 0)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_derived_data_product(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='Instrument DP', temporal_domain=tdom.dump(), spatial_domain=sdom.dump())
dp_id = self.dpsc_cli.create_data_product(dp, stream_definition_id=ctd_stream_def_id)
self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Make the derived data product
simple_stream_def_id = self.pubsubcli.create_stream_definition(name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time','temp'])
tempwat_dp = DataProduct(name='TEMPWAT')
tempwat_dp_id = self.dpsc_cli.create_data_product(tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id)
self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id)
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
rdt['pressure'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True)
tempwat_dataset_id = tempwat_dataset_ids[0]
granule = self.data_retriever.retrieve(tempwat_dataset_id, delivery_format=simple_stream_def_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
self.assertEquals(set(rdt.fields), set(['time','temp']))
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug("The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'")
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name,'DP1')
self.assertEquals(data_product_object.description,'some new dp')
log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
dataset_modified.clear()
rdt['time'] = np.arange(20,40)
publisher.publish(rdt.to_granule())
self.assertFalse(dataset_modified.wait(2))
self.dpsc_cli.activate_data_product_persistence(dp_id)
dataset_modified.clear()
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt['time'], np.arange(40))
dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True)
self.assertEquals(len(dataset_ids), 1)
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
def test_lookup_values(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsubcli.create_stream_definition('lookup', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, stream_def_id)
data_product = DataProduct(name='lookup data product')
tdom, sdom = time_series_domain()
data_product.temporal_domain = tdom.dump()
data_product.spatial_domain = sdom.dump()
data_product_id = self.dpsc_cli.create_data_product(data_product, stream_definition_id=stream_def_id)
self.addCleanup(self.dpsc_cli.delete_data_product, data_product_id)
data_producer = DataProducer(name='producer')
data_producer.producer_context = DataProcessProducerContext()
data_producer.producer_context.configuration['qc_keys'] = ['offset_document']
data_producer_id, _ = self.rrclient.create(data_producer)
self.addCleanup(self.rrclient.delete, data_producer_id)
assoc,_ = self.rrclient.create_association(subject=data_product_id, object=data_producer_id, predicate=PRED.hasDataProducer)
self.addCleanup(self.rrclient.delete_association, assoc)
document_keys = self.damsclient.list_qc_references(data_product_id)
self.assertEquals(document_keys, ['offset_document'])
svm = StoredValueManager(self.container)
svm.stored_value_cas('offset_document', {'offset_a':2.0})
self.dpsc_cli.activate_data_product_persistence(data_product_id)
dataset_ids, _ = self.rrclient.find_objects(subject=data_product_id, predicate=PRED.hasDataset, id_only=True)
dataset_id = dataset_ids[0]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temp'] = [20.]
granule = rdt.to_granule()
stream_ids, _ = self.rrclient.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True)
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['temp'], rdt2['temp'])
np.testing.assert_array_almost_equal(rdt2['calibrated'], np.array([22.0]))
svm.stored_value_cas('updated_document', {'offset_a':3.0})
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ep = EventPublisher(event_type=OT.ExternalReferencesUpdatedEvent)
ep.publish_event(origin=data_product_id, reference_keys=['updated_document'])
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [1]
rdt['temp'] = [20.]
granule = rdt.to_granule()
gevent.sleep(2) # Yield so that the event goes through
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt2['temp'],np.array([20.,20.]))
np.testing.assert_array_almost_equal(rdt2['calibrated'], np.array([22.0,23.0]))
class TestDataProductProvenance(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(
node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(
RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def test_get_data_product_provenance_report(self):
#Create a test device
device_obj = Device(name='Device1', description='test instrument site')
device_id, _ = self.rrclient.create(device_obj)
self.addCleanup(self.rrclient.delete, device_id)
#Create a test DataProduct
data_product1_obj = DataProduct(name='DataProduct1',
description='test data product 1')
data_product1_id, _ = self.rrclient.create(data_product1_obj)
self.addCleanup(self.rrclient.delete, data_product1_id)
#Create a test DataProcess
data_process_obj = DataProcess(name='DataProcess',
description='test data process')
data_process_id, _ = self.rrclient.create(data_process_obj)
self.addCleanup(self.rrclient.delete, data_process_id)
#Create a second test DataProduct
data_product2_obj = DataProduct(name='DataProduct2',
description='test data product 2')
data_product2_id, _ = self.rrclient.create(data_product2_obj)
self.addCleanup(self.rrclient.delete, data_product2_id)
#Create a test DataProducer
data_producer_obj = DataProducer(name='DataProducer',
description='test data producer')
data_producer_id, rev = self.rrclient.create(data_producer_obj)
#Link the DataProcess to the second DataProduct manually
assoc_id, _ = self.rrclient.create_association(
subject=data_process_id,
predicate=PRED.hasInputProduct,
object=data_product2_id)
self.addCleanup(self.rrclient.delete_association, assoc_id)
# Register the instrument and process. This links the device and the data process
# with their own producers
self.damsclient.register_instrument(device_id)
self.addCleanup(self.damsclient.unregister_instrument, device_id)
self.damsclient.register_process(data_process_id)
self.addCleanup(self.damsclient.unregister_process, data_process_id)
#Manually link the first DataProduct with the test DataProducer
assoc_id, _ = self.rrclient.create_association(
subject=data_product1_id,
predicate=PRED.hasDataProducer,
object=data_producer_id)
#Get the DataProducer linked to the DataProcess (created in register_process above)
#Associate that with with DataProduct1's DataProducer
data_process_producer_ids, _ = self.rrclient.find_objects(
subject=data_process_id,
predicate=PRED.hasDataProducer,
object_type=RT.DataProducer,
id_only=True)
assoc_id, _ = self.rrclient.create_association(
subject=data_process_producer_ids[0],
predicate=PRED.hasParent,
object=data_producer_id)
self.addCleanup(self.rrclient.delete_association, assoc_id)
#Get the DataProducer linked to the Device (created in register_instrument
#Associate that with the DataProcess's DataProducer
device_producer_ids, _ = self.rrclient.find_objects(
subject=device_id,
predicate=PRED.hasDataProducer,
object_type=RT.DataProducer,
id_only=True)
assoc_id, _ = self.rrclient.create_association(
subject=data_producer_id,
predicate=PRED.hasParent,
object=device_producer_ids[0])
#Create the links between the Device, DataProducts, DataProcess, and all DataProducers
self.damsclient.assign_data_product(input_resource_id=device_id,
data_product_id=data_product1_id)
self.addCleanup(self.damsclient.unassign_data_product, device_id,
data_product1_id)
self.damsclient.assign_data_product(input_resource_id=data_process_id,
data_product_id=data_product2_id)
self.addCleanup(self.damsclient.unassign_data_product, data_process_id,
data_product2_id)
#Traverse through the relationships to get the links between objects
res = self.dpmsclient.get_data_product_provenance_report(
data_product2_id)
#Make sure there are four keys
self.assertEqual(len(res.keys()), 4)
parent_count = 0
config_count = 0
for v in res.itervalues():
if 'parent' in v:
parent_count += 1
if 'config' in v:
config_count += 1
#Make sure there are three parents and four configs
self.assertEqual(parent_count, 3)
self.assertEqual(config_count, 4)
@unittest.skip('This test is obsolete with new framework')
def test_get_provenance(self):
#create a deployment with metadata and an initial site and device
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(
instrument_site_obj, "")
log.debug('test_get_provenance: new instrument_site_id id = %s ',
str(instrument_site_id))
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
log.debug('test_get_provenance: new InstrumentModel id = %s ',
str(instModel_id))
self.omsclient.assign_instrument_model_to_instrument_site(
instModel_id, instrument_site_id)
# Create InstrumentAgent
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict')
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations=[parsed_config])
try:
instAgent_id = self.imsclient.create_instrument_agent(
instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
log.debug('test_get_provenance:new InstrumentAgent id = %s',
instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
try:
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
log.debug(
"test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
#-------------------------------
# Create CTD Parsed data product
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(
name='parsed', parameter_dictionary_id=pdict_id)
log.debug(
'test_get_provenance:Creating new CDM data product with a stream definition'
)
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_parsed_data_product = self.dpmsclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_parsed_data_product)
self.dpmsclient.activate_data_product_persistence(
data_product_id=ctd_parsed_data_product)
#-------------------------------
# create a data product for the site to pass the OMS check.... we need to remove this check
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain=tdom,
spatial_domain=sdom)
log_data_product_id = self.dpmsclient.create_data_product(
dp_obj, parsed_stream_def_id)
#-------------------------------
# Deploy instrument device to instrument site
#-------------------------------
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id,
deployment_id)
self.imsclient.deploy_instrument_device(instDevice_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ",
str(deployment_id))
self.omsclient.activate_deployment(deployment_id)
inst_device_objs, _ = self.rrclient.find_objects(
subject=instrument_site_id,
predicate=PRED.hasDevice,
object_type=RT.InstrumetDevice,
id_only=False)
log.debug("test_create_deployment: deployed device: %s ",
str(inst_device_objs[0]))
#-------------------------------
# Create the agent instance
#-------------------------------
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug(
"TestDataProductProvenance: create data process definition ctd_L0_all"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new ctd_L0_all data process definition: %s" %
ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug(
"TestDataProductProvenance: create data process definition CTDL1ConductivityTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new CTDL1ConductivityTransform data process definition: %s"
% ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug(
"TestDataProductProvenance: create data process definition CTDL1PressureTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new CTDL1PressureTransform data process definition: %s"
% ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug(
"TestDataProductProvenance: create data process definition CTDL1TemperatureTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new CTDL1TemperatureTransform data process definition: %s"
% ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug(
"TestDataProductProvenance: create data process definition SalinityTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new SalinityTransform data process definition: %s"
% ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug(
"TestDataProductProvenance: create data process definition DensityTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new DensityTransform data process definition: %s"
% ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id,
ctd_L0_all_dprocdef_id,
binding='conductivity')
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id,
ctd_L0_all_dprocdef_id,
binding='pressure')
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id,
ctd_L0_all_dprocdef_id,
binding='temperature')
log.debug(
"TestDataProductProvenance: create output data product L0 conductivity"
)
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(
ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
log.debug(
"TestDataProductProvenance: create output data product L0 pressure"
)
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id)
log.debug(
"TestDataProductProvenance: create output data product L0 temperature"
)
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(
ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(
name='L1_conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l1_conductivity_id,
ctd_L1_conductivity_dprocdef_id,
binding='conductivity')
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(
name='L1_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l1_pressure_id,
ctd_L1_pressure_dprocdef_id,
binding='pressure')
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(
name='L1_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l1_temperature_id,
ctd_L1_temperature_dprocdef_id,
binding='temperature')
log.debug(
"TestDataProductProvenance: create output data product L1 conductivity"
)
ctd_l1_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(
ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
log.debug(
"TestDataProductProvenance: create output data product L1 pressure"
)
ctd_l1_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(
ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id)
log.debug(
"TestDataProductProvenance: create output data product L1 temperature"
)
ctd_l1_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(
ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(
name='L2_salinity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_salinity_id,
ctd_L2_salinity_dprocdef_id,
binding='salinity')
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(
name='L2_Density', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_density_id,
ctd_L2_density_dprocdef_id,
binding='density')
log.debug(
"TestDataProductProvenance: create output data product L2 Salinity"
)
ctd_l2_salinity_output_dp_obj = IonObject(
RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(
ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id)
log.debug(
"TestDataProductProvenance: create output data product L2 Density")
# ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
# name='L2_Density',
# description='transform output pressure',
# temporal_domain = tdom,
# spatial_domain = sdom)
#
# ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
# outgoing_stream_l2_density_id,
# parameter_dictionary)
contactInfo = ContactInformation()
contactInfo.individual_names_given = "Bill"
contactInfo.individual_name_family = "Smith"
contactInfo.street_address = "111 First St"
contactInfo.city = "San Diego"
contactInfo.email = "*****@*****.**"
contactInfo.phones = ["858-555-6666"]
contactInfo.country = "USA"
contactInfo.postal_code = "92123"
ctd_l2_density_output_dp_obj = IonObject(
RT.DataProduct,
name='L2_Density',
description='transform output pressure',
contacts=[contactInfo],
iso_topic_category="my_iso_topic_category_here",
quality_control_level="1",
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(
ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug(
"TestDataProductProvenance: create L0 all data_process start")
try:
input_data_products = [ctd_parsed_data_product]
output_data_products = [
ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id,
ctd_l0_temperature_output_dp_id
]
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=ctd_L0_all_dprocdef_id,
in_data_product_ids=input_data_products,
out_data_product_ids=output_data_products)
#activate only this data process just for coverage
self.dataprocessclient.activate_data_process(
ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment,
name='deviceLookupTable',
content=base64.encodestring(contents),
keywords=['DataProcessInput'],
attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(
ctd_l0_all_data_process_id, att)
log.info(
'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s',
deviceAttachment)
log.debug(
"TestDataProductProvenance: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug(
"TestDataProductProvenance: create L1 Conductivity data_process start"
)
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=ctd_L1_conductivity_dprocdef_id,
in_data_product_ids=[ctd_l0_conductivity_output_dp_id],
out_data_product_ids=[ctd_l1_conductivity_output_dp_id])
self.dataprocessclient.activate_data_process(
l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug(
"TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=ctd_L1_pressure_dprocdef_id,
in_data_product_ids=[ctd_l0_pressure_output_dp_id],
out_data_product_ids=[ctd_l1_pressure_output_dp_id])
self.dataprocessclient.activate_data_process(
l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug(
"TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=ctd_L1_temperature_dprocdef_id,
in_data_product_ids=[ctd_l0_temperature_output_dp_id],
out_data_product_ids=[ctd_l1_temperature_output_dp_id])
self.dataprocessclient.activate_data_process(
l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug(
"TestDataProductProvenance: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=ctd_L2_salinity_dprocdef_id,
in_data_product_ids=[
ctd_l1_conductivity_output_dp_id,
ctd_l1_pressure_output_dp_id,
ctd_l1_temperature_output_dp_id
],
out_data_product_ids=[ctd_l2_salinity_output_dp_id])
self.dataprocessclient.activate_data_process(
l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug(
"TestDataProductProvenance: create L2_Density data_process start")
try:
in_dp_ids = [
ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id,
ctd_l1_temperature_output_dp_id
]
out_dp_ids = [ctd_l2_density_output_dp_id]
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=ctd_L2_density_dprocdef_id,
in_data_product_ids=in_dp_ids,
out_data_product_ids=out_dp_ids)
self.dataprocessclient.activate_data_process(
l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('iaclient', name=ResourceAgentClient._get_agent_process_id(instDevice_id, process=FakeProcess())
# print 'activate_instrument: got ia client %s', self._ia_client
# log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
self.dataprocessclient.deactivate_data_process(
l2_density_all_data_process_id)
self.dataprocessclient.deactivate_data_process(
l2_salinity_all_data_process_id)
self.dataprocessclient.deactivate_data_process(
l1_temperature_all_data_process_id)
self.dataprocessclient.deactivate_data_process(
l1_pressure_data_process_id)
self.dataprocessclient.deactivate_data_process(
l1_conductivity_data_process_id)
self.dataprocessclient.deactivate_data_process(
ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the provenance info for the ctd density data product
#-------------------------------
provenance_dict = self.dpmsclient.get_data_product_provenance(
ctd_l2_density_output_dp_id)
log.debug("TestDataProductProvenance: provenance_dict %s",
str(provenance_dict))
#validate that products are represented
self.assertTrue(provenance_dict[str(ctd_l1_conductivity_output_dp_id)])
self.assertTrue(provenance_dict[str(ctd_l0_conductivity_output_dp_id)])
self.assertTrue(provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertTrue(provenance_dict[str(ctd_l1_temperature_output_dp_id)])
self.assertTrue(provenance_dict[str(ctd_l0_temperature_output_dp_id)])
density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertEquals(density_dict['producer'],
[l2_density_all_data_process_id])
#-------------------------------
# Retrieve the extended resource for this data product
#-------------------------------
extended_product = self.dpmsclient.get_data_product_extension(
ctd_l2_density_output_dp_id)
self.assertEqual(1, len(extended_product.data_processes))
self.assertEqual(3, len(extended_product.process_input_data_products))
# log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list))
# log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes))
# log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products))
# log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value))
#-------------------------------
# Retrieve the extended resource for this data process
#-------------------------------
extended_process_def = self.dataprocessclient.get_data_process_definition_extension(
ctd_L0_all_dprocdef_id)
# log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def))
# log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes))
# log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products))
self.assertEqual(1, len(extended_process_def.data_processes))
self.assertEqual(3,
len(extended_process_def.output_stream_definitions))
self.assertEqual(3, len(extended_process_def.data_products)
) #one list because of one data process
#-------------------------------
# Request the xml report
#-------------------------------
results = self.dpmsclient.get_data_product_provenance_report(
ctd_l2_density_output_dp_id)
print results
#-------------------------------
# Cleanup
#-------------------------------
self.dpmsclient.delete_data_product(ctd_parsed_data_product)
self.dpmsclient.delete_data_product(log_data_product_id)
self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
print 'started services'
@unittest.skip('this test just for debugging setup')
def test_just_the_setup(self):
return
def test_resources_associations(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(
any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
platform_agent_instance_id, _ = self.RR.create(
any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(instrument_agent_id, PRED.hasInstance,
instrument_agent_instance_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance,
instrument_agent_instance_id)
#self.RR.create_association(instrument_device_id, PRED.hasSensor, sensor_device_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer,
data_producer_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_agent_id, PRED.hasInstance,
platform_agent_instance_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance,
platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasInstrument,
instrument_device_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel,
sensor_model_id)
sensor_model_id #is only a target
class TestIntDataAcquisitionManagementService(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataAcquisitionManagementService
self.client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
def tearDown(self):
pass
#@unittest.skip('Not done yet.')
def test_data_source_ops(self):
# test creating a new data source
print 'Creating new data source'
datasource_obj = IonObject(RT.DataSource,
name='DataSource1',
description='instrument based new source' ,
data_source_type='sbe37')
try:
ds_id = self.client.create_data_source(datasource_obj)
except BadRequest as ex:
self.fail("failed to create new data source: %s" %ex)
print 'new data source id = ', ds_id
# test reading a non-existent data source
print 'reading non-existent data source'
try:
dp_obj = self.client.read_data_source('some_fake_id')
except NotFound as ex:
pass
else:
self.fail("non-existing data source was found during read: %s" %dp_obj)
# update a data source (tests read also)
print 'Updating data source'
# first get the existing data source object
try:
datasource_obj = self.client.read_data_source(ds_id)
except NotFound as ex:
self.fail("existing data source was not found during read")
else:
pass
# now tweak the object
datasource_obj.description = 'the very first data source'
# now write the dp back to the registry
try:
update_result = self.client.update_data_source(datasource_obj)
except NotFound as ex:
self.fail("existing data source was not found during update")
except Conflict as ex:
self.fail("revision conflict exception during data source update")
#else:
# self.assertTrue(update_result == True)
# now get the data source back to see if it was updated
try:
datasource_obj = self.client.read_data_source(ds_id)
except NotFound as ex:
self.fail("existing data source was not found during read")
else:
pass
self.assertTrue(datasource_obj.description == 'the very first data source')
# now 'delete' the data source
print "deleting data source"
try:
delete_result = self.client.force_delete_data_source(ds_id)
except NotFound as ex:
self.fail("existing data source was not found during delete")
#self.assertTrue(delete_result == True)
# now try to get the deleted dp object
try:
dp_obj = self.client.read_data_source(ds_id)
except NotFound as ex:
pass
else:
self.fail("deleted data source was found during read")
# now try to delete the already deleted data source object
print "deleting non-existing data source"
try:
delete_result = self.client.delete_data_source(ds_id)
except NotFound as ex:
pass
else:
self.fail("non-existing data source was found during delete")
def test_register_instrument(self):
# set up initial instrument to register
instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product')
instrument_id, rev = self.rrclient.create(instrument_obj)
self.base_register_instrument(instrument_id)
def test_register_platform(self):
# set up initial instrument to register
platform_obj = IonObject(RT.PlatformDevice, name='Plat1',description='a platform that is creating the data product')
platform_id, rev = self.rrclient.create(platform_obj)
#@unittest.skip('Not done yet.')
def base_register_instrument(self, instrument_id):
# Register an instrument as a data producer in coordination with DM PubSub: create stream, register and create producer object
dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product')
dataproduct_id, rev = self.rrclient.create(dataproduct_obj)
# test registering a new data producer
try:
ds_id = self.client.register_instrument(instrument_id)
except BadRequest as ex:
self.fail("failed to create new data producer: %s" %ex)
print 'new data producer id = ', ds_id
# test assigning a data product to an instrument, creating the stream for the product
try:
self.client.assign_data_product(instrument_id, dataproduct_id)
self.client.assign_data_product_source(dataproduct_id, instrument_id)
except BadRequest as ex:
self.fail("failed to assign data product to data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to assign data product to data producer: %s" %ex)
assocs = self.rrclient.find_associations(dataproduct_id, PRED.hasSource, instrument_id)
if not assocs or len(assocs) == 0:
self.fail("failed to assign data product to data producer")
# test UNassigning a data product from instrument, deleting the stream for the product
try:
self.client.unassign_data_product(instrument_id, dataproduct_id)
self.client.unassign_data_product_source(dataproduct_id, instrument_id)
except BadRequest as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
assocs = self.rrclient.find_associations(dataproduct_id, PRED.hasSource, instrument_id)
if assocs:
self.fail("failed to unassign data product to data producer")
# test UNregistering a new data producer
try:
ds_id = self.client.unregister_instrument(instrument_id)
except NotFound as ex:
self.fail("failed to unregister instrument producer: %s" %ex)
def test_register_external_data_set(self):
# Register an external data set as a data producer in coordination with DM PubSub: create stream, register and create producer object
# set up initial instrument to register
ext_dataset_obj = IonObject(RT.ExternalDataset, name='DataSet1',description='an external data feed')
ext_dataset_id, rev = self.rrclient.create(ext_dataset_obj)
dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product')
dataproduct_id, rev = self.rrclient.create(dataproduct_obj)
# test registering a new external data set
try:
ds_id = self.client.register_external_data_set(ext_dataset_id)
except BadRequest as ex:
self.fail("failed to create new data producer: %s" %ex)
print 'new data producer id = ', ds_id
# test assigning a data product to an ext_dataset_id, creating the stream for the product
try:
self.client.assign_data_product(ext_dataset_id, dataproduct_id)
except BadRequest as ex:
self.fail("failed to assign data product to data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to assign data product to data producer: %s" %ex)
# test UNassigning a data product from ext_dataset_id, deleting the stream for the product
try:
self.client.unassign_data_product(ext_dataset_id, dataproduct_id)
except BadRequest as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
# test UNregistering a external data set
try:
ds_id = self.client.unregister_external_data_set(ext_dataset_id)
except NotFound as ex:
self.fail("failed to unregister instrument producer: %s" %ex)
#@unittest.skip('not ready')
def test_eoi_resources(self):
#
# test creating a new data provider
#
print 'Creating new external_data_provider'
dataprovider_obj = IonObject(RT.ExternalDataProvider,
name='ExtDataProvider1',
description='external data provider ')
try:
dataprovider_id = self.client.create_external_data_provider(dataprovider_obj)
except BadRequest as ex:
self.fail("failed to create new data provider: %s" %ex)
print 'new data provider id = ', dataprovider_id
#
# test creating a new data source
#
print 'Creating new data source'
datasource_obj = IonObject(RT.DataSource,
name='DataSource1',
description='data source ',
data_source_type='DAP')
try:
datasource_id = self.client.create_data_source(datasource_obj)
except BadRequest as ex:
self.fail("failed to create new data source: %s" %ex)
print 'new data source id = ', datasource_id
#
# test creating a new data source model
#
print 'Creating new data source model'
datamodel_obj = IonObject(RT.DataSourceModel,
name='DataSourceModel1',
description='data source model')
try:
datamodel_id = self.client.create_data_source_model(datamodel_obj)
except BadRequest as ex:
self.fail("failed to create new data source model: %s" %ex)
print 'new data source model id = ', datamodel_id
#
# test creating a new external data set
#
print 'Creating new external data set'
dataset_obj = IonObject(RT.ExternalDataset,
name='ExternalDataSet1',
description='external data set ')
try:
extdataset_id = self.client.create_external_dataset(dataset_obj)
except BadRequest as ex:
self.fail("failed to create new external data set: %s" %ex)
print 'new external data set id = ', extdataset_id
#
# test creating a new dataset agent instance
#
print 'Creating new external data agent '
datasetagent_obj = IonObject(RT.ExternalDatasetAgent,
name='ExternalDatasetAgent1',
description='external data agent ')
try:
datasetagent_id = self.client.create_external_dataset_agent(datasetagent_obj)
except BadRequest as ex:
self.fail("failed to create new external dataset agent: %s" %ex)
print 'new external data agent id = ', datasetagent_id
#
# test creating a new datasource agent instance
#
print 'Creating new data source agent '
datasourceagent_obj = IonObject(RT.DataSourceAgent,
name='DataSourceAgent1',
description=' DataSource agent ')
try:
datasource_agent_id = self.client.create_data_source_agent(datasourceagent_obj)
except BadRequest as ex:
self.fail("failed to create new external datasource agent: %s" %ex)
print 'new external data agent id = ', datasource_agent_id
#
# test creating a new dataset agent instance
#
print 'Creating new external dataset agent instance'
datasetagentinstance_obj = IonObject(RT.ExternalDatasetAgentInstance,
name='ExternalDatasetAgentInstance1',
description='external dataset agent instance ')
try:
datasetagentinstance_id = self.client.create_external_dataset_agent_instance(datasetagentinstance_obj, datasetagent_id)
except BadRequest as ex:
self.fail("failed to create new external dataset agent instance: %s" %ex)
print 'new external data agent instance id = ', datasetagentinstance_id
#
# test creating a new datasource agent instance
#
print 'Creating new data source agent '
datasourceagentinstance_obj = IonObject(RT.DataSourceAgentInstance,
name='ExternalDataSourceAgentInstance1',
description='external DataSource agent instance ')
try:
datasource_agent_instance_id = self.client.create_data_source_agent_instance(datasourceagentinstance_obj)
except BadRequest as ex:
self.fail("failed to create new external datasource agent instance: %s" %ex)
print 'new external data agent id = ', datasource_agent_instance_id
#
# test assign / unassign
#
self.client.unassign_data_source_from_external_data_provider(datasource_id, dataprovider_id)
self.client.unassign_data_source_from_data_model(datasource_id, datamodel_id)
self.client.unassign_external_dataset_from_data_source(extdataset_id, datasource_id)
#
# test read
#
try:
dp_obj = self.client.read_external_data_provider(dataprovider_id)
except NotFound as ex:
self.fail("existing data provicer was not found during read")
else:
pass
try:
dp_obj = self.client.read_data_source(datasource_id)
except NotFound as ex:
self.fail("existing data source was not found during read")
else:
pass
#
# test delete
#
try:
self.client.delete_external_data_provider(dataprovider_id)
self.client.delete_data_source(datasource_id)
self.client.delete_external_dataset(extdataset_id)
self.client.delete_data_source_model(datamodel_id)
self.client.delete_external_dataset_agent(datasetagent_id)
self.client.delete_data_source_agent_instance(datasource_agent_instance_id)
self.client.force_delete_external_data_provider(dataprovider_id)
self.client.force_delete_data_source(datasource_id)
self.client.force_delete_external_dataset(extdataset_id)
self.client.force_delete_data_source_model(datamodel_id)
self.client.force_delete_external_dataset_agent(datasetagent_id)
self.client.force_delete_data_source_agent_instance(datasource_agent_instance_id)
except NotFound as ex:
self.fail("existing data product was not found during delete")
# test reading a non-existent data product
print 'reading non-existent data product'
try:
bad_obj = self.client.read_external_data_provider('some_fake_id')
except NotFound as ex:
pass
else:
self.fail("non-existing data product was found during read: %s" %bad_obj)
def make_grt_parser(self):
return self.client.create_parser(Parser(name='grt', description='', module='ion.util.parsers.global_range_test', method='grt_parser', config=None))
@unittest.skip("Deprecated")
def test_qc_attachment(self):
instrument_device = InstrumentDevice(name='whatever')
instrument_device_id,_ = self.rrclient.create(instrument_device)
self.addCleanup(self.rrclient.delete, instrument_device_id)
self.client.register_instrument(instrument_device_id)
self.addCleanup(self.client.unregister_instrument, instrument_device_id)
dp = DataProduct(name='instrument output')
dp_id,_ = self.rrclient.create(dp)
self.addCleanup(self.rrclient.delete, dp_id)
parser_id = self.make_grt_parser()
attachment = Attachment(name='qc ref', attachment_type=AttachmentType.REFERENCE,content=global_range_test_document, context=ReferenceAttachmentContext(parser_id=parser_id))
att_id = self.rrclient.create_attachment(dp_id, attachment)
self.addCleanup(self.rrclient.delete_attachment, att_id)
attachment2 = Attachment(name='qc ref2', attachment_type=AttachmentType.REFERENCE, content=global_range_test_document2, context=ReferenceAttachmentContext(parser_id=parser_id))
att2_id = self.rrclient.create_attachment(dp_id, attachment2)
self.addCleanup(self.rrclient.delete_attachment, att2_id)
self.client.assign_data_product(instrument_device_id, dp_id)
self.addCleanup(self.client.unassign_data_product, instrument_device_id, dp_id)
svm = StoredValueManager(self.container)
doc = svm.read_value('grt_CE01ISSM-MF005-01-CTDBPC999_TEMPWAT')
np.testing.assert_array_almost_equal(doc['grt_min_value'], -2.)
class IndexManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.ims_cli = IndexManagementServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
self.index_name = 'test_index'
def test_create_datastore_index(self):
index_name = self.index_name
ims_cli = self.ims_cli
rr_cli = self.rr_cli
options = SearchOptions()
options.attribute_match = ['test_field']
index_id = ims_cli.create_index(
name=index_name,
content_type=IndexManagementService.DATASTORE_INDEX,
options=options)
index_result = self.rr_cli.read(index_id)
self.assertIsInstance(index_result, ElasticSearchIndex)
self.assertTrue(index_result.name == index_name)
#======================================
# Clean up
#======================================
rr_cli.delete(index_id)
def test_read_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
index = ims_cli.read_index(index_id)
self.assertIsInstance(index, Index)
self.assertTrue(index.name == index_name)
rr_cli.delete(index_id)
def test_delete_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
ims_cli.delete_index(index_id)
with self.assertRaises(NotFound):
rr_cli.delete(index_id)
def test_update_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
index = ims_cli.read_index(index_id)
index.name = 'another'
ims_cli.update_index(index)
index = rr_cli.read(index_id)
self.assertTrue(index.name == 'another')
def test_find_indexes(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
#======================================
# Index Pool
#======================================
indexes = [
Index(name='first'),
Index(name='second'),
Index(name='third')
]
id_pool = list()
for index in indexes:
id_pool.append(rr_cli.create(index)[0])
index_id = ims_cli.find_indexes(index_name='second')
index = ims_cli.read_index(index_id)
self.assertTrue(index.name == 'second')
#======================================
# Clean up
#======================================
for index_id in id_pool:
rr_cli.delete(index_id)
def test_create_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
with self.assertRaises(BadRequest):
ims_cli.create_collection('failing_collection')
resources = [Resource(), Resource(), Resource()]
resources = [rr_cli.create(i)[0] for i in resources]
collection_id = ims_cli.create_collection('working_collection',
resources)
collection = rr_cli.read(collection_id)
collection_resources = ims_cli.list_collection_resources(collection_id,
id_only=True)
self.assertTrue(
set(collection_resources) == set(resources),
'%s != %s' % (set(collection_resources), set(resources)))
def test_read_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
collection = Collection(name='working_collection')
collection_id, _ = rr_cli.create(collection)
collection = ims_cli.read_collection(collection_id)
self.assertTrue(collection.name == 'working_collection')
def test_update_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
collection = Collection(name='useful_collection')
collection_id, _ = rr_cli.create(collection)
collection = rr_cli.read(collection_id)
collection.name = 'nub'
ims_cli.update_collection(collection)
collection = rr_cli.read(collection_id)
self.assertTrue(collection.name == 'nub')
def test_delete_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
res = Resource()
res_id, rev = rr_cli.create(res)
collection_id = ims_cli.create_collection(name='test_collection',
resources=[res_id])
ims_cli.delete_collection(collection_id)
with self.assertRaises(NotFound):
rr_cli.read(collection_id)
def test_list_collection_resources(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
#========================================
# Resource Pool
#========================================
resources = [
InformationResource(name='bean_counter'),
InformationResource(name='lunar_rock'),
InformationResource('aperature'),
InformationResource('lemons')
]
resources = [rr_cli.create(i)[0] for i in resources]
collection = Collection(name='park_bench')
collection_id = ims_cli.create_collection(name='park_bench',
resources=resources)
retval = ims_cli.list_collection_resources(collection_id, id_only=True)
retval.sort()
resources.sort()
self.assertTrue(retval == resources, '%s != %s' % (retval, resources))
def test_find_collection(self):
res_id, _ = self.rr_cli.create(Resource(name='test_res'))
collection_id = self.ims_cli.create_collection('test', [res_id])
retval = self.ims_cli.find_collection(collection_name='test')
self.assertTrue(retval[0] == collection_id)
retval = self.ims_cli.find_collection(resource_ids=[res_id])
self.assertTrue(retval[0] == collection_id)
class TestResourceRegistryAttachments(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
print 'started services'
def test_resource_registry_blob_sanity(self):
resource_id, _ = self.RR.create(IonObject(RT.Resource, name="foo"))
MY_CONTENT = "the quick brown fox etc etc etc"
#save
att_id = self.RR.create_attachment(resource_id, IonObject(RT.Attachment,
name="test.txt",
content=MY_CONTENT,
content_type="text/plain",
keywords=["test1", "test2"],
attachment_type=AttachmentType.BLOB))
#load
attachment = self.RR.read_attachment(att_id, include_content=True)
self.assertEqual("test.txt", attachment.name)
self.assertEqual("text/plain", attachment.content_type)
self.assertIn("test1", attachment.keywords)
self.assertIn("test2", attachment.keywords)
#content has changed; it's base64-encoded from what we put in
self.assertEqual(MY_CONTENT, attachment.content)
obj = self.RR.read(resource_id)
self.assertEqual(obj.name, "foo")
obj.name = "TheDudeAbides"
obj = self.RR.update(obj)
obj = self.RR.read(resource_id)
self.assertEqual(obj.name, "TheDudeAbides")
att = self.RR.find_attachments(resource_id)
self.assertNotEqual(att, None)
actor_identity_obj = IonObject("ActorIdentity", name="name")
actor_identity_obj_id, actor_identity_obj_rev = self.RR.create(actor_identity_obj)
user_info_obj = IonObject("UserInfo", name="name")
user_info_obj_id, user_info_obj_rev = self.RR.create(user_info_obj)
assoc_id, assoc_rev = self.RR.create_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
self.assertNotEqual(assoc_id, None)
find_assoc = self.RR.find_associations(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
self.assertTrue(find_assoc[0]._id == assoc_id)
subj = self.RR.find_subjects(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True)
res_obj1 = self.RR.read_object(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo)
self.assertEquals(res_obj1._id, user_info_obj_id)
self.RR.delete_association(assoc_id)
self.RR.delete_attachment(att_id)
self.RR.delete(resource_id)
class MissionManager(object):
"""
Coordinating class for integration of mission execution with platform agent.
"""
def __init__(self, pa):
"""
Called by platform agent upon its initialization so there is a driver
already created and configured.
@param pa The associated platform agent object to access the
elements handled by this helper.
"""
self._agent = pa
self._platform_id = pa._platform_id
# mission_id -> MissionScheduler mapping:
self._running_missions = {}
log.debug('%r: [mm] MissionManager created', self._platform_id)
self._provider_id = self._agent._provider_id
self._actor_id = self._agent._actor_id
log.debug('%r: [xa] provider_id=%r actor_id=%r', self._platform_id, self._provider_id, self._actor_id)
# ctx = self._agent.get_context()
# self._actor_id = ctx.get('ion-actor-id', None) if ctx else None
# log.debug('[xa] actor_id=%r', self._actor_id)
if self._actor_id is None:
log.warn('%r: [xa] actor_id is None', self._platform_id)
# _exaccess: resource_id -> {'commitment_id': id, 'mission_ids': [mission_id, ...]}:
# the agents we have acquired exclusive access to. We remove the actual exclusive
# access when there are no more associated mission_id's for a given resource_id.
self._exaccess = {}
self.ORG = OrgManagementServiceProcessClient(process=self._agent)
self.RR = ResourceRegistryServiceClient()
# TODO what's the correct way to obtain the actor header? the following is
# working but likely because the same call is done in
# base_test_platform_agent_with_rsn for the IMS.start_platform_agent_instance call
self._actor_header = get_system_actor_header()
log.debug('%r: [xa] actor_header=%s', self._platform_id, self._actor_header)
def get_number_of_running_missions(self):
return len(self._running_missions)
def load_mission(self, mission_id, mission_yml):
"""
Loads a mission as preparation prior to its actual execution.
@param mission_id
@param mission_yml
@return (mission_loader, mission_scheduler, instrument_objs) arguments
for subsequence call to run_mission
@raise BadRequest if mission_id is already running or there's any
problem loading the mission
"""
if mission_id in self._running_missions:
raise BadRequest('run_mission: mission_id=%r is already running', mission_id)
try:
mission_loader, mission_scheduler, instrument_objs = \
self._create_mission_scheduler(mission_id, mission_yml)
except Exception as ex:
msg = '%r: [mm] run_mission: mission_id=%r _create_mission_scheduler exception: %s' % (
self._platform_id, mission_id, ex)
log.exception(msg)
raise BadRequest(msg)
return mission_id, mission_loader, mission_scheduler, instrument_objs
def run_mission(self, mission_id, mission_loader, mission_scheduler, instrument_objs):
"""
Runs a mission returning to caller when the execution is completed.
Parameters as returned by load_mission.
"""
if mission_id in self._running_missions:
raise BadRequest('run_mission: mission_id=%r is already running', mission_id)
self._running_missions[mission_id] = mission_scheduler
log.debug('%r: [mm] starting mission_id=%r (#running missions=%s)', self._platform_id,
mission_id, len(self._running_missions))
try:
mission_scheduler.run_mission()
except Exception as ex:
log.exception('%r: [mm] run_mission mission_id=%r', self._platform_id, mission_id)
finally:
del self._running_missions[mission_id]
# remove exclusive access:
mission_entries = mission_loader.mission_entries
for mission_entry in mission_entries:
instrument_ids = mission_entry.get('instrument_id', [])
for instrument_id in instrument_ids:
if instrument_id in instrument_objs:
resource_id = instrument_objs[instrument_id].resource_id
self._remove_exclusive_access(instrument_id, resource_id, mission_id)
log.debug('%r: [mm] completed mission_id=%r (#running missions=%s)', self._platform_id,
mission_id, len(self._running_missions))
def abort_mission(self, mission_id):
if mission_id not in self._running_missions:
raise BadRequest('abort_mission: invalid mission_id=%r', mission_id)
mission_scheduler = self._running_missions[mission_id]
try:
mission_scheduler.abort_mission()
return None
except Exception as ex:
log.exception('%r: [mm] abort_mission', self._platform_id)
return ex
finally:
del self._running_missions[mission_id]
def destroy(self):
"""
Called by platform agent when it is reset.
Aborts all ongoing missions if any. Any errors are logged out.
"""
mission_ids = self._running_missions.keys()
nn = len(mission_ids)
if nn:
log.debug('%r: [mm] MissionManager.destroy called. Aborting %d ongoing missions...',
self._platform_id, nn)
for mission_id in mission_ids:
try:
self.abort_mission(mission_id)
except Exception as ignored:
pass
self._running_missions = {}
else:
log.debug('%r: [mm] MissionManager.destroy called. No ongoing missions executing.',
self._platform_id)
############
# private
############
def _create_mission_scheduler(self, mission_id, mission_yml):
"""
- loads the mission
- verifies instruments associated
- gets exclusive access to those instruments
- creates MissionScheduler
@param mission_id
@param mission_yml
@return (mission_loader, mission_scheduler, instrument_objs)
@raise Exception if no stable ID is found for an instrument; or
the first exception while requesting exclusive
access to a child instrument (in this case, all other successful
such requests, if any, are reverted).
"""
log.debug('%r: [mm] _create_mission_scheduler: mission_id=%r', self._platform_id, mission_id)
mission_loader = MissionLoader(self._agent)
mission_loader.load_mission(mission_id, mission_yml)
self._mission_entries = mission_loader.mission_entries
if log.isEnabledFor(logging.DEBUG):
log.debug('%r: [mm] _create_mission_scheduler: _ia_clients=\n%s', self._platform_id,
self._agent._pp.pformat(self._agent._ia_clients))
# {stable_id: obj, ...} objects of valid running instruments:
instrument_objs = {}
for (instrument_id, obj) in self._agent._ia_clients.iteritems():
if isinstance(obj, dict): # dict means it's valid instrument.
# get first "PRE:*" ID from obj.alt_ids:
pres = [alt_id for alt_id in obj.alt_ids if alt_id.startswith('PRE:')]
if not pres:
raise Exception('%r: No stable ID found for instrument_id=%r. alt_ids=%s' % (
self._platform_id, instrument_id, obj.alt_ids))
stable_id = pres[0]
log.debug('%r: [mm] _create_mission_scheduler: instrument_id=%r, stable_id=%r,'
' resource_id=%r', self._platform_id, instrument_id, stable_id, obj.resource_id)
instrument_objs[stable_id] = obj
# {stable_id: client, ...} dict for scheduler
instruments_for_scheduler = dict((stable_id, obj.ia_client) for
stable_id, obj in instrument_objs.iteritems())
mission_entries = mission_loader.mission_entries
# get all involved instruments referenced in the mission:
instrument_ids = set()
for mission_entry in mission_entries:
for instrument_id in mission_entry.get('instrument_id', []):
if instrument_id in instrument_objs:
instrument_ids.add(instrument_id)
else:
raise Exception('%r: No stable ID found for instrument_id=%r referenced'
' in mission, mission_id=%r' % (self._platform_id,
instrument_id, mission_id))
# get exclusive access to those instruments. If any one fails,
# rollback and raise that first exception:
instrument_ids_ok = set()
exception = None
for instrument_id in instrument_ids:
resource_id = instrument_objs[instrument_id].resource_id
try:
self._get_exclusive_access(instrument_id, resource_id, mission_id)
instrument_ids_ok.add(instrument_id)
except Exception as ex:
exception = ex
log.warn('%r: [xa] _create_mission_scheduler: exclusive access request to'
' resource_id=%r, instrument_id=%r failed: %s', self._platform_id,
resource_id, instrument_id, exception)
break
if exception:
if len(instrument_ids_ok):
log.warn('%r: [xa] _create_mission_scheduler: reverting exclusive access to the resources: %s',
self._platform_id, instrument_ids_ok)
for instrument_id in instrument_ids_ok:
resource_id = instrument_objs[instrument_id].resource_id
try:
self._remove_exclusive_access(instrument_id, resource_id, mission_id)
except Exception as ex:
# just log warning an continue
log.warn('%r: [xa] exception while reverting exclusive access to resource_id=%r, '
'instrument_id=%r: %s', self._platform_id, resource_id, instrument_id, ex)
raise exception
mission_scheduler = MissionScheduler(self._agent,
instruments_for_scheduler,
mission_entries)
log.debug('%r: [mm] _create_mission_scheduler: MissionScheduler created. entries=%s',
self._platform_id, mission_entries)
return mission_loader, mission_scheduler, instrument_objs
def _get_exclusive_access(self, instrument_id, resource_id, mission_id):
"""
Gets exclusive access for the given resource_id. The actual request is
only done once for the same resource_id, but we keep track of the
associated mission_id such that the exclusive access is removed when
no missions remain referencing the resource_id.
@param instrument_id for logging
@param resource_id
@param mission_id
"""
# check if we already have exclusive access to resource_id:
if resource_id in self._exaccess:
mission_ids = self._exaccess[resource_id]['mission_ids']
if mission_id in mission_ids:
log.debug('%r: [xa] resource_id=%r already with exclusive access, mission_id=%r',
self._platform_id, resource_id, mission_id)
else:
mission_ids.append(mission_id)
log.debug('%r: [xa] resource_id=%r already with exclusive access from '
'previous call with mission_id=%r', self._platform_id, resource_id, mission_ids[0])
return
log.debug('%r: [xa] _get_exclusive_access: instrument_id=%r resource_id=%r, actor_id=%r, provider=%r',
self._platform_id, instrument_id, resource_id, self._actor_id, self._provider_id)
# TODO proper handling of BadRequest exception upon failure to obtain
# exclusive access. For now, just logging ghe exception.
try:
commitment_id = self._do_get_exclusive_access(instrument_id, resource_id)
self._exaccess[resource_id] = dict(commitment_id=commitment_id,
mission_ids=[mission_id])
except BadRequest:
log.exception('%r: [xa] _get_exclusive_access: instrument_id=%r resource_id=%r, mission_id=%r',
self._platform_id, instrument_id, resource_id, mission_id)
def _do_get_exclusive_access(self, instrument_id, resource_id):
"""
Gets exclusive access to a given resource.
@return commitment_id
"""
# TODO Needs review
from interface.objects import NegotiationTypeEnum
neg_type = NegotiationTypeEnum.INVITATION
neg_obj = IonObject(RT.Negotiation, negotiation_type=neg_type)
negotiation_id, _ = self.RR.create(neg_obj)
# TODO determine appropriate expiration. Could it be without expiration
# given that the exclusive access will be removed explicitly upon
# termination (normal or abnormal) of the the mission?
expiration = int(get_ion_ts()) + 20 * 60 * 1000
# the SAP for the acquire resource exclusively proposal:
arxp = IonObject(OT.AcquireResourceExclusiveProposal,
consumer=self._actor_id,
resource_id=resource_id,
provider=self._provider_id,
expiration=str(expiration),
negotiation_id=negotiation_id)
# we are initially opting for only "phase 2" -- just acquire_resource:
commitment_id = self.ORG.acquire_resource(arxp, headers=self._actor_header)
log.debug('%r: [xa] AcquireResourceExclusiveProposal: instrument_id=%r resource_id=%s -> '
'commitment_id=%s', self._platform_id, instrument_id, resource_id, commitment_id)
return commitment_id
# #####################################################################
# # with "negotiation" it seems it would involve something like the
# # following (see coi/../test_gobernance.py):
# # 1- negotiate a base acquire resource proposal
# # 2- negotiate the exclusive acquire resource proposal
#
# # acquire resource proposal
# arp = IonObject(OT.AcquireResourceProposal,
# consumer=self._actor_id,
# resource_id=resource_id,
# provider=self._provider_id)
# arp_response = self.ORG.negotiate(arp, headers=self._actor_header)
# log.debug('%r: [xa] _get_exclusive_access/AcquireResourceProposal: '
# 'resource_id=%s -> arp_response=%s', self._platform_id, resource_id, arp_response)
#
# arxp_response = self.ORG.negotiate(arxp, headers=self._actor_header)
# log.debug('%r: [xa] _get_exclusive_access/AcquireResourceExclusiveProposal: '
# 'resource_id=%s -> arxp_response=%s', self._platform_id, resource_id, arxp_response)
def _remove_exclusive_access(self, instrument_id, resource_id, mission_id):
"""
Removes the exclusive access for the given resource_id. The actual
removal is only done if there are no more missions associated.
@param instrument_id for logging
@param resource_id
@param mission_id
"""
if not resource_id in self._exaccess:
log.warn('%r: [xa] not associated with exclusive access resource_id=%r instrument_id=%r',
self._platform_id, resource_id, instrument_id)
return
mission_ids = self._exaccess[resource_id]['mission_ids']
if not mission_id in mission_ids:
log.warn('%r: [xa] not associated with exclusive access resource_id=%r instrument_id=%r',
self._platform_id, resource_id, instrument_id)
return
mission_ids.remove(mission_id)
if len(mission_ids) > 0:
log.debug('%r: [xa] exclusive access association removed: resource_id=%r -> mission_id=%r, '
'instrument_id=%r', self._platform_id, resource_id, mission_id, instrument_id)
return
# no more mission_ids associated, so release the exclusive access:
commitment_id = self._exaccess[resource_id]['commitment_id']
del self._exaccess[resource_id]
self._do_remove_exclusive_access(commitment_id, instrument_id, resource_id)
def _do_remove_exclusive_access(self, commitment_id, instrument_id, resource_id):
"""
Does the actual release of the exclusive access.
@param commitment_id commitment to the released
@param instrument_id associated ID for logging purposes
@param resource_id associated resource ID for logging purposes
"""
# TODO: any exception below is just logged out; need different handling?
try:
ret = self.ORG.release_commitment(commitment_id)
log.debug('%r: [xa] exclusive access removed: resource_id=%r instrument_id=%r: '
'ORG.release_commitment(commitment_id=%r) returned=%r', self._platform_id,
resource_id, instrument_id, commitment_id, ret)
except Exception as ex:
log.exception('%r: [xa] resource_id=%r instrument_id=%r: ORG.release_commitment(commitment_id=%r)',
self._platform_id, resource_id, instrument_id, commitment_id)
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(TestActivateInstrumentIntegration.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete, index)
IndexManagementService._es_call(es.index_delete, index)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_notification(self,
user_name='',
instrument_id='',
product_id=''):
#--------------------------------------------------------------------------------------
# Make notification request objects
#--------------------------------------------------------------------------------------
notification_request_1 = NotificationRequest(
name='notification_1',
origin=instrument_id,
origin_type="instrument",
event_type='ResourceLifecycleEvent')
notification_request_2 = NotificationRequest(
name='notification_2',
origin=product_id,
origin_type="data product",
event_type='DetectionEvent')
#--------------------------------------------------------------------------------------
# Create a user and get the user_id
#--------------------------------------------------------------------------------------
user = UserInfo()
user.name = user_name
user.contact.email = '*****@*****.**' % user_name
user_id, _ = self.rrclient.create(user)
#--------------------------------------------------------------------------------------
# Create notification
#--------------------------------------------------------------------------------------
self.usernotificationclient.create_notification(
notification=notification_request_1, user_id=user_id)
self.usernotificationclient.create_notification(
notification=notification_request_2, user_id=user_id)
log.debug(
"test_activateInstrumentSample: create_user_notifications user_id %s",
str(user_id))
return user_id
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def _check_computed_attributes_of_extended_instrument(
self, expected_instrument_device_id='', extended_instrument=None):
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(
extended_instrument.computed.last_data_received_datetime,
ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
self.assertIsInstance(
extended_instrument.computed.power_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.communications_status_roll_up,
ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.location_status_roll_up,
ComputedIntValue)
# the following assert will not work without elasticsearch.
#self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
# Verify the computed attribute for user notification requests
self.assertEqual(
1,
len(extended_instrument.computed.user_notification_requests.value))
notifications = extended_instrument.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(expected_instrument_device_id, notification.origin)
self.assertEqual("instrument", notification.origin_type)
self.assertEqual('ResourceLifecycleEvent', notification.event_type)
def _check_computed_attributes_of_extended_product(
self, expected_data_product_id='', extended_data_product=None):
self.assertEqual(expected_data_product_id, extended_data_product._id)
log.debug("extended_data_product.computed: %s",
extended_data_product.computed)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(
extended_data_product.computed.product_download_size_estimated,
ComputedFloatValue)
self.assertIsInstance(
extended_data_product.computed.number_active_subscriptions,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_url,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.stored_data_size,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.recent_granules,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.parameters,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.recent_events,
ComputedEventListValue)
self.assertIsInstance(extended_data_product.computed.provenance,
ComputedDictValue)
self.assertIsInstance(
extended_data_product.computed.user_notification_requests,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.active_user_subscriptions,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.past_user_subscriptions,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.last_granule,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.is_persisted,
ComputedIntValue)
self.assertIsInstance(
extended_data_product.computed.data_contents_updated,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.data_datetime,
ComputedListValue)
# exact text here keeps changing to fit UI capabilities. keep assertion general...
self.assertIn(
'ok',
extended_data_product.computed.last_granule.value['quality_flag'])
self.assertEqual(
2, len(extended_data_product.computed.data_datetime.value))
notifications = extended_data_product.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(expected_data_product_id, notification.origin)
self.assertEqual("data product", notification.origin_type)
self.assertEqual('DetectionEvent', notification.event_type)
@attr('LOCOINT')
#@unittest.skip('refactoring')
@unittest.skipIf(not use_es, 'No ElasticSearch')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint': {'receive': {'timeout': 90}}})
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', instModel_id)
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='raw')
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict')
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations=[raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config,
alerts=[])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'raw', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0])
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# setup notifications for the device and parsed data product
user_id_1 = self._create_notification(user_name='user_1',
instrument_id=instDevice_id,
product_id=data_product_id1)
#---------- Create notifications for another user and verify that we see different computed subscriptions for the two users ---------
user_id_2 = self._create_notification(user_name='user_2',
instrument_id=instDevice_id,
product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
log.debug('Data product streams2 = %s', str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id2 = %s', dataset_ids[0])
self.raw_dataset = dataset_ids[0]
#elastic search debug
es_indexes, _ = self.container.resource_registry.find_resources(
restype='ElasticSearchIndex')
log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes])
log.debug('Bootstrap %s', CFG.bootstrap.use_es)
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id, ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
#log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s",
str(self._ia_client))
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s", str(retval))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.INACTIVE, state)
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s",
str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.IDLE, state)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active command %s",
str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s", str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.STOPPED, state)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.IDLE, state)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
for i in xrange(10):
retval = self._ia_client.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from sample %s",
str(retval))
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s",
str(reply))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data_raw, Granule)
rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw)
log.debug("RDT raw: %s", str(rdt_raw.pretty_print()))
self.assertIn('raw', rdt_raw)
raw_vals = rdt_raw['raw']
all_raw = "".join(raw_vals)
# look for 't' entered after a prompt -- ">t"
t_commands = all_raw.count(">t")
if 10 != t_commands:
log.error("%s raw_vals: ", len(raw_vals))
for i, r in enumerate(raw_vals):
log.error("raw val %s: %s", i, [r])
self.fail("Expected 10 't' strings in raw_vals, got %s" %
t_commands)
else:
log.debug("%s raw_vals: ", len(raw_vals))
for i, r in enumerate(raw_vals):
log.debug("raw val %s: %s", i, [r])
replay_data_parsed = self.dataretrieverclient.retrieve(
self.parsed_dataset)
self.assertIsInstance(replay_data_parsed, Granule)
rdt_parsed = RecordDictionaryTool.load_from_granule(replay_data_parsed)
log.debug("test_activateInstrumentSample: RDT parsed: %s",
str(rdt_parsed.pretty_print()))
self.assertIn('temp', rdt_parsed)
temp_vals = rdt_parsed['temp']
pressure_vals = rdt_parsed['pressure']
if 10 != len(temp_vals):
log.error("%s temp_vals: %s", len(temp_vals), temp_vals)
self.fail("Expected 10 temp_vals, got %s" % len(temp_vals))
log.debug("l4-ci-sa-rq-138")
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
#--------------------------------------------------------------------------------
# Get the extended data product to see if it contains the granules
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id1, user_id=user_id_1)
def poller(extended_product):
return len(extended_product.computed.user_notification_requests.
value) == 1
poll(poller, extended_product, timeout=30)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id1,
extended_data_product=extended_product)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_1)
#--------------------------------------------------------------------------------
# For the second user, check the extended data product and the extended intrument
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id2, user_id=user_id_2)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id2,
extended_data_product=extended_product)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_2)
self._check_computed_attributes_of_extended_instrument(
expected_instrument_device_id=instDevice_id,
extended_instrument=extended_instrument)
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(data_product_id1)
self.dpclient.delete_data_product(data_product_id2)
class TestResourceRegistry(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to bank service
self.resource_registry_service = ResourceRegistryServiceClient()
@unittest.skip('Represents a bug in storage/retrieval')
def test_tuple_in_dict(self):
# create a resource with a tuple saved in a dict
transform_obj = IonObject(RT.Transform)
transform_obj.configuration = {}
transform_obj.configuration["tuple"] = ('STRING', )
transform_id, _ = self.resource_registry_service.create(transform_obj)
# read the resource back
returned_transform_obj = self.resource_registry_service.read(
transform_id)
self.assertEqual(transform_obj.configuration["tuple"],
returned_transform_obj.configuration["tuple"])
def test_basics(self):
# Sequence all the tests so that we can save numerous system start and stops
self._do_test_crud()
self._do_test_read_mult()
self._do_test_lifecycle()
self._do_test_attach()
self._do_test_association()
self._do_test_find_resources()
self._do_test_find_objects_mult()
def _do_test_crud(self):
# Some quick registry tests
# Can't call new with fields that aren't defined in the object's schema
with self.assertRaises(TypeError) as cm:
IonObject("UserInfo", name="name", foo="bar")
self.assertTrue(cm.exception.message ==
"__init__() got an unexpected keyword argument 'foo'")
# Can't call new with fields that aren't defined in the object's schema
with self.assertRaises(TypeError) as cm:
IonObject("UserInfo", {"name": "name", "foo": "bar"})
self.assertTrue(cm.exception.message ==
"__init__() got an unexpected keyword argument 'foo'")
# Can't call new with fields that aren't defined in the object's schema
with self.assertRaises(TypeError) as cm:
IonObject("UserInfo", {"name": "name"}, foo="bar")
self.assertTrue(cm.exception.message ==
"__init__() got an unexpected keyword argument 'foo'")
# Instantiate an object
obj = IonObject("UserInfo", name="name")
# Can set attributes that aren't in the object's schema
with self.assertRaises(AttributeError) as cm:
setattr(obj, "foo", "bar")
self.assertTrue(
cm.exception.message == "'UserInfo' object has no attribute 'foo'")
# Cam't call update with object that hasn't been persisted
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.update(obj)
self.assertTrue(
cm.exception.message.startswith(
"Object does not have required '_id' or '_rev' attribute"))
# Persist object and read it back
obj_id, obj_rev = self.resource_registry_service.create(obj)
read_obj = self.resource_registry_service.read(obj_id)
# Cannot create object with _id and _rev fields pre-set
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create(read_obj)
# Update object
read_obj.name = "John Doe"
self.resource_registry_service.update(read_obj)
# Update should fail with revision mismatch
with self.assertRaises(Conflict) as cm:
self.resource_registry_service.update(read_obj)
# Re-read and update object
read_obj = self.resource_registry_service.read(obj_id)
self.resource_registry_service.update(read_obj)
# Delete object
self.resource_registry_service.delete(obj_id)
# Make sure read, update and delete report error
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.read(obj_id)
self.assertTrue(cm.exception.message.startswith("Object with id"))
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.update(read_obj)
self.assertTrue(cm.exception.message.startswith("Object with id"))
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.delete(obj_id)
self.assertTrue(cm.exception.message.startswith("Object with id"))
# Owner creation tests
user = IonObject("ActorIdentity", name='user')
uid, _ = self.resource_registry_service.create(user)
inst = IonObject("InstrumentDevice", name='instrument')
iid, _ = self.resource_registry_service.create(
inst, headers={'ion-actor-id': str(uid)})
ids, _ = self.resource_registry_service.find_objects(iid,
PRED.hasOwner,
RT.ActorIdentity,
id_only=True)
self.assertEquals(len(ids), 1)
assoc = self.resource_registry_service.read(ids[0])
self.resource_registry_service.delete(iid)
with self.assertRaises(NotFound) as ex:
assoc = self.resource_registry_service.read(iid)
def _do_test_read_mult(self):
test_resource1_id, _ = self.resource_registry_service.create(
Resource(name='test1'))
test_resource2_id, _ = self.resource_registry_service.create(
Resource(name='test2'))
res_list = [test_resource1_id, test_resource2_id]
objects = self.resource_registry_service.read_mult(res_list)
for o in objects:
self.assertIsInstance(o, Resource)
self.assertTrue(o._id in res_list)
def _do_test_lifecycle(self):
# Lifecycle tests
att = IonObject("InstrumentDevice", name='mine', description='desc')
rid, rev = self.resource_registry_service.create(att)
att1 = self.resource_registry_service.read(rid)
self.assertEquals(att1.name, att.name)
self.assertEquals(att1.lcstate, LCS.DRAFT)
self.assertEquals(att1.availability, AS.PRIVATE)
new_state = self.resource_registry_service.execute_lifecycle_transition(
rid, LCE.PLAN)
self.assertEquals(new_state, lcstate(LCS.PLANNED, AS.PRIVATE))
att2 = self.resource_registry_service.read(rid)
self.assertEquals(att2.lcstate, LCS.PLANNED)
self.assertEquals(att2.availability, AS.PRIVATE)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.execute_lifecycle_transition(
rid, LCE.UNANNOUNCE)
self.assertTrue(
"type=InstrumentDevice, lcstate=PLANNED_PRIVATE has no transition for event unannounce"
in cm.exception.message)
new_state = self.resource_registry_service.execute_lifecycle_transition(
rid, LCE.DEVELOP)
self.assertEquals(new_state, lcstate(LCS.DEVELOPED, AS.PRIVATE))
with self.assertRaises(BadRequest):
self.resource_registry_service.execute_lifecycle_transition(
resource_id=rid, transition_event='NONE##')
self.resource_registry_service.set_lifecycle_state(
rid, lcstate(LCS.INTEGRATED, AS.PRIVATE))
att1 = self.resource_registry_service.read(rid)
self.assertEquals(att1.lcstate, LCS.INTEGRATED)
self.assertEquals(att1.availability, AS.PRIVATE)
def _do_test_attach(self):
binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82"
# Owner creation tests
instrument = IonObject("InstrumentDevice", name='instrument')
iid, _ = self.resource_registry_service.create(instrument)
att = Attachment(content=binary, attachment_type=AttachmentType.BLOB)
aid1 = self.resource_registry_service.create_attachment(iid, att)
att1 = self.resource_registry_service.read_attachment(
aid1, include_content=True)
self.assertEquals(binary, att1.content)
import base64
att = Attachment(content=base64.encodestring(binary),
attachment_type=AttachmentType.ASCII)
aid2 = self.resource_registry_service.create_attachment(iid, att)
att1 = self.resource_registry_service.read_attachment(
aid2, include_content=True)
self.assertEquals(binary, base64.decodestring(att1.content))
att_ids = self.resource_registry_service.find_attachments(iid,
id_only=True)
self.assertEquals(att_ids, [aid1, aid2])
att_ids = self.resource_registry_service.find_attachments(
iid, id_only=True, descending=True)
self.assertEquals(att_ids, [aid2, aid1])
att_ids = self.resource_registry_service.find_attachments(
iid, id_only=True, descending=True, limit=1)
self.assertEquals(att_ids, [aid2])
atts = self.resource_registry_service.find_attachments(
iid, id_only=False, include_content=True, limit=1)
self.assertEquals(atts[0].content, binary)
self.resource_registry_service.delete_attachment(aid1)
att_ids = self.resource_registry_service.find_attachments(iid,
id_only=True)
self.assertEquals(att_ids, [aid2])
self.resource_registry_service.delete_attachment(aid2)
att_ids = self.resource_registry_service.find_attachments(iid,
id_only=True)
self.assertEquals(att_ids, [])
def _do_test_association(self):
# Instantiate ActorIdentity object
actor_identity_obj = IonObject("ActorIdentity", name="name")
actor_identity_obj_id, actor_identity_obj_rev = self.resource_registry_service.create(
actor_identity_obj)
read_actor_identity_obj = self.resource_registry_service.read(
actor_identity_obj_id)
# Instantiate UserInfo object
user_info_obj = IonObject("UserInfo", name="name")
user_info_obj_id, user_info_obj_rev = self.resource_registry_service.create(
user_info_obj)
read_user_info_obj = self.resource_registry_service.read(
user_info_obj_id)
# Test create failures
with self.assertRaises(AttributeError) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.bogus, user_info_obj_id)
self.assertTrue(cm.exception.message == "bogus")
# Predicate not provided
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, None, user_info_obj_id)
self.assertTrue(
cm.exception.message == "Association must have all elements set")
# Subject id or object not provided
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
None, PRED.hasInfo, user_info_obj_id)
self.assertTrue(
cm.exception.message == "Association must have all elements set")
# Object id or object not provided
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, None)
self.assertTrue(
cm.exception.message == "Association must have all elements set")
# Bad subject id
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.create_association(
"bogus", PRED.hasInfo, user_info_obj_id)
self.assertTrue(
cm.exception.message == "Object with id bogus does not exist.")
# Bad object id
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, "bogus")
self.assertTrue(
cm.exception.message == "Object with id bogus does not exist.")
# _id missing from subject
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
actor_identity_obj, PRED.hasInfo, user_info_obj_id)
self.assertTrue(cm.exception.message.startswith("Subject id"))
# _id missing from object
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, user_info_obj)
self.assertTrue(cm.exception.message.startswith("Object id"))
# Wrong subject type
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
user_info_obj_id, PRED.hasInfo, user_info_obj_id)
self.assertTrue(cm.exception.message ==
"Illegal subject type UserInfo for predicate hasInfo")
# Wrong object type
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, actor_identity_obj_id)
self.assertTrue(
cm.exception.message ==
"Illegal object type ActorIdentity for predicate hasInfo")
# Create two different association types between the same subject and predicate
assoc_id1, assoc_rev1 = self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
# Read object, subject
res_obj1 = self.resource_registry_service.read_object(
actor_identity_obj_id, PRED.hasInfo, RT.UserInfo)
self.assertEquals(res_obj1._id, user_info_obj_id)
res_obj1 = self.resource_registry_service.read_object(
actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, id_only=True)
self.assertEquals(res_obj1, user_info_obj_id)
res_obj2 = self.resource_registry_service.read_subject(
RT.ActorIdentity, PRED.hasInfo, user_info_obj_id)
self.assertEquals(res_obj2._id, actor_identity_obj_id)
res_obj2 = self.resource_registry_service.read_subject(
RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, id_only=True)
self.assertEquals(res_obj2, actor_identity_obj_id)
# Search for associations (good cases)
ret1 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
ret2 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo)
ret3 = self.resource_registry_service.find_associations(
None, PRED.hasInfo)
self.assertTrue(len(ret1) == len(ret2) == len(ret3))
self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id)
ret1 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, None, False)
ret2 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo, id_only=False)
ret3 = self.resource_registry_service.find_associations(
predicate=PRED.hasInfo, id_only=False)
self.assertTrue(ret1 == ret2 == ret3)
# Search for associations (good cases)
ret1 = self.resource_registry_service.find_associations(
read_actor_identity_obj, PRED.hasInfo, read_user_info_obj)
ret2 = self.resource_registry_service.find_associations(
read_actor_identity_obj, PRED.hasInfo)
ret3 = self.resource_registry_service.find_associations(
None, PRED.hasInfo)
self.assertTrue(len(ret1) == len(ret2) == len(ret3))
self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id)
ret1 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo, read_user_info_obj, None,
True)
ret2 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo, id_only=True)
ret3 = self.resource_registry_service.find_associations(
predicate=PRED.hasInfo, id_only=True)
self.assertTrue(ret1 == ret2 == ret3)
# Search for associations (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_associations(None, None, None)
self.assertIn("Illegal parameters", cm.exception.message)
# Find subjects (good cases)
subj_ret1 = self.resource_registry_service.find_subjects(
RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True)
subj_ret2 = self.resource_registry_service.find_subjects(
RT.ActorIdentity, PRED.hasInfo, read_user_info_obj, True)
self.assertTrue(len(subj_ret1) == len(subj_ret2))
self.assertTrue(subj_ret1[0] == subj_ret2[0])
self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id)
subj_ret3 = self.resource_registry_service.find_subjects(
None, PRED.hasInfo, user_info_obj_id, True)
subj_ret4 = self.resource_registry_service.find_subjects(
None, None, read_user_info_obj, True)
self.assertTrue(len(subj_ret3) == len(subj_ret4))
self.assertTrue(subj_ret3[0] == subj_ret4[0])
self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id)
subj_ret5 = self.resource_registry_service.find_subjects(
None, PRED.hasInfo, user_info_obj_id, False)
subj_ret6 = self.resource_registry_service.find_subjects(
None, None, read_user_info_obj, False)
self.assertTrue(len(subj_ret5) == len(subj_ret6))
self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id)
self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id)
# Find subjects (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_subjects(None, None, None)
self.assertTrue(cm.exception.message == "Must provide object")
with self.assertRaises(AttributeError) as cm:
self.resource_registry_service.find_subjects(
RT.UserCredentials, PRED.bogus, user_info_obj_id, True)
self.assertTrue(cm.exception.message == "bogus")
ret = self.resource_registry_service.find_subjects(
RT.UserInfo, PRED.hasCredentials, user_info_obj_id, True)
self.assertTrue(len(ret[0]) == 0)
ret = self.resource_registry_service.find_subjects(
RT.UserCredentials, PRED.hasInfo, user_info_obj_id, True)
self.assertTrue(len(ret[0]) == 0)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_subjects(
RT.UserCredentials, PRED.hasInfo, user_info_obj, True)
self.assertTrue(
cm.exception.message == "Object id not available in object")
# Find objects (good cases)
subj_ret1 = self.resource_registry_service.find_objects(
actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, True)
subj_ret2 = self.resource_registry_service.find_objects(
read_actor_identity_obj, PRED.hasInfo, RT.UserInfo, True)
self.assertTrue(len(subj_ret1) == len(subj_ret2))
self.assertTrue(subj_ret1[0] == subj_ret2[0])
self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id)
subj_ret3 = self.resource_registry_service.find_objects(
actor_identity_obj_id, PRED.hasInfo, None, True)
subj_ret4 = self.resource_registry_service.find_objects(
actor_identity_obj_id, None, None, True)
self.assertTrue(len(subj_ret3) == len(subj_ret4))
self.assertTrue(subj_ret3[0] == subj_ret4[0])
self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id)
subj_ret5 = self.resource_registry_service.find_objects(
actor_identity_obj_id, PRED.hasInfo, None, False)
subj_ret6 = self.resource_registry_service.find_objects(
read_actor_identity_obj, None, None, False)
self.assertTrue(len(subj_ret5) == len(subj_ret6))
self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id)
self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id)
# Find objects (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_objects(None, None, None)
self.assertTrue(cm.exception.message == "Must provide subject")
with self.assertRaises(AttributeError) as cm:
self.resource_registry_service.find_objects(
actor_identity_obj_id, PRED.bogus, RT.UserCredentials, True)
self.assertTrue(cm.exception.message == "bogus")
ret = self.resource_registry_service.find_objects(
actor_identity_obj_id, PRED.hasCredentials, RT.ActorIdentity, True)
self.assertTrue(len(ret[0]) == 0)
ret = self.resource_registry_service.find_objects(
actor_identity_obj_id, PRED.hasInfo, RT.UserCredentials, True)
self.assertTrue(len(ret[0]) == 0)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_objects(
actor_identity_obj, PRED.hasInfo, RT.UserInfo, True)
self.assertTrue(
cm.exception.message == "Object id not available in subject")
# Get association (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.get_association(None, None, None)
self.assertIn("Illegal parameters", cm.exception.message)
assoc = self.resource_registry_service.get_association(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
self.assertTrue(assoc._id == assoc_id1)
# Delete (bad cases)
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.delete_association("bogus")
self.assertTrue(
cm.exception.message == "Object with id bogus does not exist.")
# Delete other association
self.resource_registry_service.delete_association(assoc_id1)
# Delete resources
self.resource_registry_service.delete(actor_identity_obj_id)
self.resource_registry_service.delete(user_info_obj_id)
def _do_test_find_resources(self):
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_resources(
RT.UserInfo, LCS.DRAFT, "name", False)
self.assertTrue(
cm.exception.message == "find by name does not support lcstate")
ret = self.resource_registry_service.find_resources(
RT.UserInfo, None, "name", False)
self.assertEquals(len(ret[0]), 0)
# Instantiate an object
obj = IonObject("InstrumentAgentInstance", name="name")
# Persist object and read it back
obj_id, obj_rev = self.resource_registry_service.create(obj)
read_obj = self.resource_registry_service.read(obj_id)
ret = self.resource_registry_service.find_resources(
RT.InstrumentAgentInstance, None, "name", False)
self.assertEquals(len(ret[0]), 1)
self.assertEquals(ret[0][0]._id, read_obj._id)
ret = self.resource_registry_service.find_resources(
RT.InstrumentAgentInstance, LCS.DEPLOYED, None, False)
self.assertEquals(len(ret[0]), 1)
self.assertEquals(ret[0][0]._id, read_obj._id)
def _do_test_find_objects_mult(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.resource_registry_service.create(dp)
transform_id, _ = self.resource_registry_service.create(transform)
pd_id, _ = self.resource_registry_service.create(pd)
self.resource_registry_service.create_association(
subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.resource_registry_service.create_association(
subject=transform_id,
object=pd_id,
predicate=PRED.hasProcessDefinition)
results, _ = self.resource_registry_service.find_objects_mult(
subjects=[dp_id], id_only=True)
self.assertTrue(results == [transform_id])
results, _ = self.resource_registry_service.find_objects_mult(
subjects=[dp_id, transform_id], id_only=True)
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@attr('EXT')
def test_get_resource_extension(self):
#Testing multiple instrument owners
subject1 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
actor_identity_obj1 = IonObject(RT.ActorIdentity, {"name": subject1})
actor_id1, _ = self.resource_registry_service.create(
actor_identity_obj1)
user_info_obj1 = IonObject(RT.UserInfo, {"name": "Foo"})
user_info_id1, _ = self.resource_registry_service.create(
user_info_obj1)
self.resource_registry_service.create_association(
actor_id1, PRED.hasInfo, user_info_id1)
subject2 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256"
actor_identity_obj2 = IonObject(RT.ActorIdentity, {"name": subject2})
actor_id2, _ = self.resource_registry_service.create(
actor_identity_obj2)
user_info_obj2 = IonObject(RT.UserInfo, {"name": "Foo2"})
user_info_id2, _ = self.resource_registry_service.create(
user_info_obj2)
self.resource_registry_service.create_association(
actor_id2, PRED.hasInfo, user_info_id2)
test_obj = IonObject(RT.InformationResource, {"name": "TestResource"})
test_obj_id, _ = self.resource_registry_service.create(test_obj)
self.resource_registry_service.create_association(
test_obj_id, PRED.hasOwner, actor_id1)
self.resource_registry_service.create_association(
test_obj_id, PRED.hasOwner, actor_id2)
extended_resource = self.resource_registry_service.get_resource_extension(
test_obj_id, OT.ExtendedInformationResource)
self.assertEqual(test_obj_id, extended_resource._id)
self.assertEqual(len(extended_resource.owners), 2)
extended_resource_list = self.resource_registry_service.get_resource_extension(
str([user_info_id1, user_info_id2]),
OT.ExtendedInformationResource)
self.assertEqual(len(extended_resource_list), 2)
optional_args = {'user_id': user_info_id1}
extended_resource = self.resource_registry_service.get_resource_extension(
test_obj_id,
OT.TestExtendedInformationResource,
optional_args=optional_args)
self.assertEqual(test_obj_id, extended_resource._id)
self.assertEqual(len(extended_resource.owners), 2)
self.assertEqual(extended_resource.user_id, user_info_id1)
@attr('PREP')
def test_prepare_resource_support(self):
prepare_data = self.resource_registry_service.prepare_resource_support(
resource_type=RT.StreamDefinition)
self.assertEqual(prepare_data.create_request.service_name,
"resource_registry")
self.assertEqual(prepare_data.create_request.service_operation,
"create")
self.assertEqual(prepare_data.create_request.request_parameters,
{"object": "$(object)"})
self.assertEqual(prepare_data.update_request.service_name,
"resource_registry")
self.assertEqual(prepare_data.update_request.service_operation,
"update")
self.assertEqual(prepare_data.update_request.request_parameters,
{"object": "$(object)"})
res_id, _ = self.resource_registry_service.create(
prepare_data.resource)
prepare_data = self.resource_registry_service.prepare_resource_support(
resource_type=RT.StreamDefinition, resource_id=res_id)
prepare_data.resource.name = "test_stream_def"
prepare_data.resource.stream_type = "test_type"
stream_def_id, _ = self.resource_registry_service.update(
prepare_data.resource)
#def ion_object_encoder(obj):
# return obj.__dict__
#print simplejson.dumps(prepare_data, default=ion_object_encoder, indent=2)
stream_def = self.resource_registry_service.read(stream_def_id)
self.assertEqual(stream_def.name, prepare_data.resource.name)
self.assertEqual(stream_def.stream_type,
prepare_data.resource.stream_type)
class IngestionManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.ingestion_management = IngestionManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.ingest_name = 'basic'
self.exchange = 'testdata'
@staticmethod
def clean_subscriptions():
ingestion_management = IngestionManagementServiceClient()
pubsub = PubsubManagementServiceClient()
rr = ResourceRegistryServiceClient()
ingestion_config_ids = ingestion_management.list_ingestion_configurations(
id_only=True)
for ic in ingestion_config_ids:
subscription_ids, assocs = rr.find_objects(
subject=ic, predicate=PRED.hasSubscription, id_only=True)
for subscription_id, assoc in zip(subscription_ids, assocs):
rr.delete_association(assoc)
try:
pubsub.deactivate_subscription(subscription_id)
except:
log.exception("Unable to decativate subscription: %s",
subscription_id)
pubsub.delete_subscription(subscription_id)
def create_ingest_config(self):
self.queue = IngestionQueue(name='test', type='testdata')
# Create the ingestion config
ingestion_config_id = self.ingestion_management.create_ingestion_configuration(
name=self.ingest_name,
exchange_point_id=self.exchange,
queues=[self.queue])
return ingestion_config_id
def test_ingestion_config_crud(self):
ingestion_config_id = self.create_ingest_config()
ingestion_config = self.ingestion_management.read_ingestion_configuration(
ingestion_config_id)
self.assertTrue(ingestion_config.name == self.ingest_name)
self.assertTrue(ingestion_config.queues[0].name == 'test')
self.assertTrue(ingestion_config.queues[0].type == 'testdata')
ingestion_config.name = 'another'
self.ingestion_management.update_ingestion_configuration(
ingestion_config)
# Create an association just to make sure that it will delete them
sub = Subscription()
sub_id, _ = self.resource_registry.create(sub)
assoc_id, _ = self.resource_registry.create_association(
subject=ingestion_config_id,
predicate=PRED.hasSubscription,
object=sub_id)
self.ingestion_management.delete_ingestion_configuration(
ingestion_config_id)
with self.assertRaises(NotFound):
self.resource_registry.read(assoc_id)
def test_list_ingestion(self):
# Create the ingest_config
config_id = self.create_ingest_config()
retval = self.ingestion_management.list_ingestion_configurations(
id_only=True)
# Nice thing about this is that it breaks if r2dm adds an ingest_config
self.assertTrue(config_id in retval)
class TestResourceRegistry(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to bank service
self.resource_registry_service = ResourceRegistryServiceClient()
@unittest.skip('Represents a bug in storage/retrieval')
def test_tuple_in_dict(self):
# create a resource with a tuple saved in a dict
transform_obj = IonObject(RT.Transform)
transform_obj.configuration = {}
transform_obj.configuration["tuple"] = ('STRING',)
transform_id, _ = self.resource_registry_service.create(transform_obj)
# read the resource back
returned_transform_obj = self.resource_registry_service.read(transform_id)
self.assertEqual(transform_obj.configuration["tuple"], returned_transform_obj.configuration["tuple"])
def test_basics(self):
# Sequence all the tests so that we can save numerous system start and stops
self._do_test_crud()
self._do_test_read_mult()
self._do_test_lifecycle()
self._do_test_attach()
self._do_test_association()
self._do_test_find_resources()
self._do_test_find_objects_mult()
def _do_test_crud(self):
# Some quick registry tests
# Can't call new with fields that aren't defined in the object's schema
with self.assertRaises(TypeError) as cm:
IonObject("UserInfo", name="name", foo="bar")
self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'")
# Can't call new with fields that aren't defined in the object's schema
with self.assertRaises(TypeError) as cm:
IonObject("UserInfo", {"name": "name", "foo": "bar"})
self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'")
# Can't call new with fields that aren't defined in the object's schema
with self.assertRaises(TypeError) as cm:
IonObject("UserInfo", {"name": "name"}, foo="bar")
self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'")
# Instantiate an object
obj = IonObject("UserInfo", name="name")
# Can set attributes that aren't in the object's schema
with self.assertRaises(AttributeError) as cm:
setattr(obj, "foo", "bar")
self.assertTrue(cm.exception.message == "'UserInfo' object has no attribute 'foo'")
# Cam't call update with object that hasn't been persisted
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.update(obj)
self.assertTrue(cm.exception.message.startswith("Object does not have required '_id' or '_rev' attribute"))
# Persist object and read it back
obj_id, obj_rev = self.resource_registry_service.create(obj)
read_obj = self.resource_registry_service.read(obj_id)
# Cannot create object with _id and _rev fields pre-set
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create(read_obj)
# Update object
read_obj.name = "John Doe"
self.resource_registry_service.update(read_obj)
# Update should fail with revision mismatch
with self.assertRaises(Conflict) as cm:
self.resource_registry_service.update(read_obj)
# Re-read and update object
read_obj = self.resource_registry_service.read(obj_id)
self.resource_registry_service.update(read_obj)
# Delete object
self.resource_registry_service.delete(obj_id)
# Make sure read, update and delete report error
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.read(obj_id)
self.assertTrue(cm.exception.message.startswith("Object with id"))
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.update(read_obj)
self.assertTrue(cm.exception.message.startswith("Object with id"))
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.delete(obj_id)
self.assertTrue(cm.exception.message.startswith("Object with id"))
# Owner creation tests
user = IonObject("ActorIdentity", name='user')
uid,_ = self.resource_registry_service.create(user)
inst = IonObject("InstrumentDevice", name='instrument')
iid,_ = self.resource_registry_service.create(inst, headers={'ion-actor-id':str(uid)})
ids,_ = self.resource_registry_service.find_objects(iid, PRED.hasOwner, RT.ActorIdentity, id_only=True)
self.assertEquals(len(ids), 1)
assoc = self.resource_registry_service.read(ids[0])
self.resource_registry_service.delete(iid)
with self.assertRaises(NotFound) as ex:
assoc = self.resource_registry_service.read(iid)
def _do_test_read_mult(self):
test_resource1_id,_ = self.resource_registry_service.create(Resource(name='test1'))
test_resource2_id,_ = self.resource_registry_service.create(Resource(name='test2'))
res_list = [test_resource1_id, test_resource2_id]
objects = self.resource_registry_service.read_mult(res_list)
for o in objects:
self.assertIsInstance(o,Resource)
self.assertTrue(o._id in res_list)
def _do_test_lifecycle(self):
# Lifecycle tests
att = IonObject("InstrumentDevice", name='mine', description='desc')
rid,rev = self.resource_registry_service.create(att)
att1 = self.resource_registry_service.read(rid)
self.assertEquals(att1.name, att.name)
self.assertEquals(att1.lcstate, LCS.DRAFT)
self.assertEquals(att1.availability, AS.PRIVATE)
new_state = self.resource_registry_service.execute_lifecycle_transition(rid, LCE.PLAN)
self.assertEquals(new_state, lcstate(LCS.PLANNED, AS.PRIVATE))
att2 = self.resource_registry_service.read(rid)
self.assertEquals(att2.lcstate, LCS.PLANNED)
self.assertEquals(att2.availability, AS.PRIVATE)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.execute_lifecycle_transition(rid, LCE.UNANNOUNCE)
self.assertTrue("type=InstrumentDevice, lcstate=PLANNED_PRIVATE has no transition for event unannounce" in cm.exception.message)
new_state = self.resource_registry_service.execute_lifecycle_transition(rid, LCE.DEVELOP)
self.assertEquals(new_state, lcstate(LCS.DEVELOPED, AS.PRIVATE))
with self.assertRaises(BadRequest):
self.resource_registry_service.execute_lifecycle_transition(
resource_id=rid, transition_event='NONE##')
self.resource_registry_service.set_lifecycle_state(rid, lcstate(LCS.INTEGRATED, AS.PRIVATE))
att1 = self.resource_registry_service.read(rid)
self.assertEquals(att1.lcstate, LCS.INTEGRATED)
self.assertEquals(att1.availability, AS.PRIVATE)
def _do_test_attach(self):
binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82"
# Owner creation tests
instrument = IonObject("InstrumentDevice", name='instrument')
iid,_ = self.resource_registry_service.create(instrument)
att = Attachment(content=binary, attachment_type=AttachmentType.BLOB)
aid1 = self.resource_registry_service.create_attachment(iid, att)
att1 = self.resource_registry_service.read_attachment(aid1, include_content=True)
self.assertEquals(binary, att1.content)
import base64
att = Attachment(content=base64.encodestring(binary), attachment_type=AttachmentType.ASCII)
aid2 = self.resource_registry_service.create_attachment(iid, att)
att1 = self.resource_registry_service.read_attachment(aid2, include_content=True)
self.assertEquals(binary, base64.decodestring(att1.content))
att_ids = self.resource_registry_service.find_attachments(iid, id_only=True)
self.assertEquals(att_ids, [aid1, aid2])
att_ids = self.resource_registry_service.find_attachments(iid, id_only=True, descending=True)
self.assertEquals(att_ids, [aid2, aid1])
att_ids = self.resource_registry_service.find_attachments(iid, id_only=True, descending=True, limit=1)
self.assertEquals(att_ids, [aid2])
atts = self.resource_registry_service.find_attachments(iid, id_only=False, include_content=True, limit=1)
self.assertEquals(atts[0].content, binary)
self.resource_registry_service.delete_attachment(aid1)
att_ids = self.resource_registry_service.find_attachments(iid, id_only=True)
self.assertEquals(att_ids, [aid2])
self.resource_registry_service.delete_attachment(aid2)
att_ids = self.resource_registry_service.find_attachments(iid, id_only=True)
self.assertEquals(att_ids, [])
def _do_test_association(self):
# Instantiate ActorIdentity object
actor_identity_obj = IonObject("ActorIdentity", name="name")
actor_identity_obj_id, actor_identity_obj_rev = self.resource_registry_service.create(actor_identity_obj)
read_actor_identity_obj = self.resource_registry_service.read(actor_identity_obj_id)
# Instantiate UserInfo object
user_info_obj = IonObject("UserInfo", name="name")
user_info_obj_id, user_info_obj_rev = self.resource_registry_service.create(user_info_obj)
read_user_info_obj = self.resource_registry_service.read(user_info_obj_id)
# Test create failures
with self.assertRaises(AttributeError) as cm:
self.resource_registry_service.create_association(actor_identity_obj_id, PRED.bogus, user_info_obj_id)
self.assertTrue(cm.exception.message == "bogus")
# Predicate not provided
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(actor_identity_obj_id, None, user_info_obj_id)
self.assertTrue(cm.exception.message == "Association must have all elements set")
# Subject id or object not provided
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(None, PRED.hasInfo, user_info_obj_id)
self.assertTrue(cm.exception.message == "Association must have all elements set")
# Object id or object not provided
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, None)
self.assertTrue(cm.exception.message == "Association must have all elements set")
# Bad subject id
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.create_association("bogus", PRED.hasInfo, user_info_obj_id)
self.assertTrue(cm.exception.message == "Object with id bogus does not exist.")
# Bad object id
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, "bogus")
self.assertTrue(cm.exception.message == "Object with id bogus does not exist.")
# _id missing from subject
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(actor_identity_obj, PRED.hasInfo, user_info_obj_id)
self.assertTrue(cm.exception.message.startswith("Subject id"))
# _id missing from object
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj)
self.assertTrue(cm.exception.message.startswith("Object id"))
# Wrong subject type
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(user_info_obj_id, PRED.hasInfo, user_info_obj_id)
self.assertTrue(cm.exception.message == "Illegal subject type UserInfo for predicate hasInfo")
# Wrong object type
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, actor_identity_obj_id)
self.assertTrue(cm.exception.message == "Illegal object type ActorIdentity for predicate hasInfo")
# Create two different association types between the same subject and predicate
assoc_id1, assoc_rev1 = self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
# Read object, subject
res_obj1 = self.resource_registry_service.read_object(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo)
self.assertEquals(res_obj1._id, user_info_obj_id)
res_obj1 = self.resource_registry_service.read_object(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, id_only=True)
self.assertEquals(res_obj1, user_info_obj_id)
res_obj2 = self.resource_registry_service.read_subject(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id)
self.assertEquals(res_obj2._id, actor_identity_obj_id)
res_obj2 = self.resource_registry_service.read_subject(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, id_only=True)
self.assertEquals(res_obj2, actor_identity_obj_id)
# Search for associations (good cases)
ret1 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo)
ret3 = self.resource_registry_service.find_associations(None, PRED.hasInfo)
self.assertTrue(len(ret1) == len(ret2) == len(ret3))
self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id)
ret1 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, None, False)
ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, id_only=False)
ret3 = self.resource_registry_service.find_associations(predicate=PRED.hasInfo, id_only=False)
self.assertTrue(ret1 == ret2 == ret3)
# Search for associations (good cases)
ret1 = self.resource_registry_service.find_associations(read_actor_identity_obj, PRED.hasInfo, read_user_info_obj)
ret2 = self.resource_registry_service.find_associations(read_actor_identity_obj, PRED.hasInfo)
ret3 = self.resource_registry_service.find_associations(None, PRED.hasInfo)
self.assertTrue(len(ret1) == len(ret2) == len(ret3))
self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id)
ret1 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, read_user_info_obj, None, True)
ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, id_only=True)
ret3 = self.resource_registry_service.find_associations(predicate=PRED.hasInfo, id_only=True)
self.assertTrue(ret1 == ret2 == ret3)
# Search for associations (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_associations(None, None, None)
self.assertIn("Illegal parameters", cm.exception.message)
# Find subjects (good cases)
subj_ret1 = self.resource_registry_service.find_subjects(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True)
subj_ret2 = self.resource_registry_service.find_subjects(RT.ActorIdentity, PRED.hasInfo, read_user_info_obj, True)
self.assertTrue(len(subj_ret1) == len(subj_ret2))
self.assertTrue(subj_ret1[0] == subj_ret2[0])
self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id)
subj_ret3 = self.resource_registry_service.find_subjects(None, PRED.hasInfo, user_info_obj_id, True)
subj_ret4 = self.resource_registry_service.find_subjects(None, None, read_user_info_obj, True)
self.assertTrue(len(subj_ret3) == len(subj_ret4))
self.assertTrue(subj_ret3[0] == subj_ret4[0])
self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id)
subj_ret5 = self.resource_registry_service.find_subjects(None, PRED.hasInfo, user_info_obj_id, False)
subj_ret6 = self.resource_registry_service.find_subjects(None, None, read_user_info_obj, False)
self.assertTrue(len(subj_ret5) == len(subj_ret6))
self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id)
self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id)
# Find subjects (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_subjects(None, None, None)
self.assertTrue(cm.exception.message == "Must provide object")
with self.assertRaises(AttributeError) as cm:
self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.bogus, user_info_obj_id, True)
self.assertTrue(cm.exception.message == "bogus")
ret = self.resource_registry_service.find_subjects(RT.UserInfo, PRED.hasCredentials, user_info_obj_id, True)
self.assertTrue(len(ret[0]) == 0)
ret = self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.hasInfo, user_info_obj_id, True)
self.assertTrue(len(ret[0]) == 0)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.hasInfo, user_info_obj, True)
self.assertTrue(cm.exception.message == "Object id not available in object")
# Find objects (good cases)
subj_ret1 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, True)
subj_ret2 = self.resource_registry_service.find_objects(read_actor_identity_obj, PRED.hasInfo, RT.UserInfo, True)
self.assertTrue(len(subj_ret1) == len(subj_ret2))
self.assertTrue(subj_ret1[0] == subj_ret2[0])
self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id)
subj_ret3 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, None, True)
subj_ret4 = self.resource_registry_service.find_objects(actor_identity_obj_id, None, None, True)
self.assertTrue(len(subj_ret3) == len(subj_ret4))
self.assertTrue(subj_ret3[0] == subj_ret4[0])
self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id)
subj_ret5 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, None, False)
subj_ret6 = self.resource_registry_service.find_objects(read_actor_identity_obj, None, None, False)
self.assertTrue(len(subj_ret5) == len(subj_ret6))
self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id)
self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id)
# Find objects (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_objects(None, None, None)
self.assertTrue(cm.exception.message == "Must provide subject")
with self.assertRaises(AttributeError) as cm:
self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.bogus, RT.UserCredentials, True)
self.assertTrue(cm.exception.message == "bogus")
ret = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasCredentials, RT.ActorIdentity, True)
self.assertTrue(len(ret[0]) == 0)
ret = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, RT.UserCredentials, True)
self.assertTrue(len(ret[0]) == 0)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_objects(actor_identity_obj, PRED.hasInfo, RT.UserInfo, True)
self.assertTrue(cm.exception.message == "Object id not available in subject")
# Get association (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.get_association(None, None, None)
self.assertIn("Illegal parameters", cm.exception.message)
assoc = self.resource_registry_service.get_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
self.assertTrue(assoc._id == assoc_id1)
# Delete (bad cases)
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.delete_association("bogus")
self.assertTrue(cm.exception.message == "Object with id bogus does not exist.")
# Delete other association
self.resource_registry_service.delete_association(assoc_id1)
# Delete resources
self.resource_registry_service.delete(actor_identity_obj_id)
self.resource_registry_service.delete(user_info_obj_id)
def _do_test_find_resources(self):
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_resources(RT.UserInfo, LCS.DRAFT, "name", False)
self.assertTrue(cm.exception.message == "find by name does not support lcstate")
ret = self.resource_registry_service.find_resources(RT.UserInfo, None, "name", False)
self.assertEquals(len(ret[0]), 0)
# Instantiate an object
obj = IonObject("InstrumentAgentInstance", name="name")
# Persist object and read it back
obj_id, obj_rev = self.resource_registry_service.create(obj)
read_obj = self.resource_registry_service.read(obj_id)
ret = self.resource_registry_service.find_resources(RT.InstrumentAgentInstance, None, "name", False)
self.assertEquals(len(ret[0]), 1)
self.assertEquals(ret[0][0]._id, read_obj._id)
ret = self.resource_registry_service.find_resources(RT.InstrumentAgentInstance, LCS.DEPLOYED, None, False)
self.assertEquals(len(ret[0]), 1)
self.assertEquals(ret[0][0]._id, read_obj._id)
def _do_test_find_objects_mult(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.resource_registry_service.create(dp)
transform_id, _ = self.resource_registry_service.create(transform)
pd_id, _ = self.resource_registry_service.create(pd)
self.resource_registry_service.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.resource_registry_service.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results, _ = self.resource_registry_service.find_objects_mult(subjects=[dp_id],id_only=True)
self.assertTrue(results == [transform_id])
results, _ = self.resource_registry_service.find_objects_mult(subjects=[dp_id, transform_id], id_only=True)
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@attr('EXT')
def test_get_resource_extension(self):
#Testing multiple instrument owners
subject1 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
actor_identity_obj1 = IonObject(RT.ActorIdentity, {"name": subject1})
actor_id1,_ = self.resource_registry_service.create(actor_identity_obj1)
user_info_obj1 = IonObject(RT.UserInfo, {"name": "Foo"})
user_info_id1,_ = self.resource_registry_service.create(user_info_obj1)
self.resource_registry_service.create_association(actor_id1, PRED.hasInfo, user_info_id1)
subject2 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256"
actor_identity_obj2 = IonObject(RT.ActorIdentity, {"name": subject2})
actor_id2,_ = self.resource_registry_service.create(actor_identity_obj2)
user_info_obj2 = IonObject(RT.UserInfo, {"name": "Foo2"})
user_info_id2,_ = self.resource_registry_service.create(user_info_obj2)
self.resource_registry_service.create_association(actor_id2, PRED.hasInfo, user_info_id2)
test_obj = IonObject(RT.InformationResource, {"name": "TestResource"})
test_obj_id,_ = self.resource_registry_service.create(test_obj)
self.resource_registry_service.create_association(test_obj_id, PRED.hasOwner, actor_id1)
self.resource_registry_service.create_association(test_obj_id, PRED.hasOwner, actor_id2)
extended_resource = self.resource_registry_service.get_resource_extension(test_obj_id, OT.ExtendedInformationResource )
self.assertEqual(test_obj_id,extended_resource._id)
self.assertEqual(len(extended_resource.owners),2)
extended_resource_list = self.resource_registry_service.get_resource_extension(str([user_info_id1,user_info_id2]), OT.ExtendedInformationResource)
self.assertEqual(len(extended_resource_list), 2)
optional_args = {'user_id': user_info_id1}
extended_resource = self.resource_registry_service.get_resource_extension(test_obj_id, OT.TestExtendedInformationResource, optional_args=optional_args )
self.assertEqual(test_obj_id,extended_resource._id)
self.assertEqual(len(extended_resource.owners),2)
self.assertEqual(extended_resource.user_id, user_info_id1)
class UserNotificationIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.unsc = UserNotificationServiceClient(node=self.container.node)
self.rrc = ResourceRegistryServiceClient(node=self.container.node)
self.imc = IdentityManagementServiceClient(node=self.container.node)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_email(self):
proc1 = self.container.proc_manager.procs_by_name['user_notification']
# Create a user and get the user_id
user = UserInfo(name = 'new_user')
user_id, _ = self.rrc.create(user)
# set up....
notification_id = self.unsc.create_email(event_type='ResourceLifecycleEvent',
event_subtype=None,
origin='Some_Resource_Agent_ID1',
origin_type=None,
user_id=user_id,
email='*****@*****.**',
mode = DeliveryMode.DIGEST,
message_header='message_header',
parser='parser',
period=1)
#------------------------------------------------------------------------------------------------------
# Setup so as to be able to get the message and headers going into the
# subscription callback method of the EmailEventProcessor
#------------------------------------------------------------------------------------------------------
# publish an event for each notification to generate the emails
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event")
msg_tuple = proc1.event_processors[notification_id].smtp_client.sentmail.get(timeout=4)
self.assertTrue(proc1.event_processors[notification_id].smtp_client.sentmail.empty())
message = msg_tuple[2]
list_lines = message.split("\n")
#-------------------------------------------------------
# parse the message body
#-------------------------------------------------------
message_dict = {}
for line in list_lines:
key_item = line.split(": ")
if key_item[0] == 'Subject':
message_dict['Subject'] = key_item[1] + key_item[2]
else:
try:
message_dict[key_item[0]] = key_item[1]
except IndexError as exc:
# these IndexError exceptions happen only because the message sometimes
# has successive /r/n (i.e. new lines) and therefore,
# the indexing goes out of range. These new lines
# can just be ignored. So we ignore the exceptions here.
pass
#-------------------------------------------------------
# make assertions
#-------------------------------------------------------
self.assertEquals(msg_tuple[1], '*****@*****.**' )
#self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS)
#self.assertEquals(message_dict['From'], ION_NOTIFICATION_EMAIL_ADDRESS)
self.assertEquals(message_dict['To'], '*****@*****.**')
self.assertEquals(message_dict['Event'].rstrip('\r'), 'ResourceLifecycleEvent')
self.assertEquals(message_dict['Originator'].rstrip('\r'), 'Some_Resource_Agent_ID1')
self.assertEquals(message_dict['Description'].rstrip('\r'), 'RLE test event')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_sms(self):
proc1 = self.container.proc_manager.procs_by_name['user_notification']
# Create a user and get the user_id
user = UserInfo(name = 'new_user')
user_id, _ = self.rrc.create(user)
# set up....
notification_id = self.unsc.create_sms(event_type='ResourceLifecycleEvent',
event_subtype=None,
origin='Some_Resource_Agent_ID1',
origin_type=None,
user_id=user_id,
phone = '401-XXX-XXXX',
provider='T-Mobile',
message_header='message_header',
parser='parser',
)
#------------------------------------------------------------------------------------------------------
# Setup so as to be able to get the message and headers going into the
# subscription callback method of the EmailEventProcessor
#------------------------------------------------------------------------------------------------------
# publish an event for each notification to generate the emails
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event")
msg_tuple = proc1.event_processors[notification_id].smtp_client.sentmail.get(timeout=4)
self.assertTrue(proc1.event_processors[notification_id].smtp_client.sentmail.empty())
message = msg_tuple[2]
list_lines = message.split("\n")
#-------------------------------------------------------
# parse the message body
#-------------------------------------------------------
message_dict = {}
for line in list_lines:
key_item = line.split(": ")
if key_item[0] == 'Subject':
message_dict['Subject'] = key_item[1] + key_item[2]
else:
try:
message_dict[key_item[0]] = key_item[1]
except IndexError as exc:
# these IndexError exceptions happen only because the message sometimes
# has successive /r/n (i.e. new lines) and therefore,
# the indexing goes out of range. These new lines
# can just be ignored. So we ignore the exceptions here.
pass
#-------------------------------------------------------
# make assertions
#-------------------------------------------------------
self.assertEquals(msg_tuple[1], '*****@*****.**' )
#self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS)
self.assertEquals(message_dict['Description'].rstrip('\r'), 'RLE test event')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_event_detection(self):
proc1 = self.container.proc_manager.procs_by_name['user_notification']
# Create a user and get the user_id
user = UserInfo(name = 'new_user')
user_id, _ = self.rrc.create(user)
# Create detection notification
dfilt = DetectionFilterConfig()
dfilt.processing['condition'] = 5
dfilt.processing['comparator'] = '>'
dfilt.processing['filter_field'] = 'voltage'
dfilt.delivery['message'] = 'I got my detection event!'
notification_id = self.unsc.create_detection_filter(event_type='ExampleDetectableEvent',
event_subtype=None,
origin='Some_Resource_Agent_ID1',
origin_type=None,
user_id=user_id,
filter_config=dfilt
)
#---------------------------------------------------------------------------------
# Create event subscription for resulting detection event
#---------------------------------------------------------------------------------
# Create an email notification so that when the DetectionEventProcessor
# detects an event and fires its own output event, this will caught by an
# EmailEventProcessor and an email will be sent to the user
notification_id_2 = self.unsc.create_email(event_type='DetectionEvent',
event_subtype=None,
origin='DetectionEventProcessor',
origin_type=None,
user_id=user_id,
email='*****@*****.**',
mode = DeliveryMode.UNFILTERED,
message_header='Detection event',
parser='parser',
period=1)
# Send event that is not detected
# publish an event for each notification to generate the emails
rle_publisher = EventPublisher("ExampleDetectableEvent")
# since the voltage field in this event is less than 5, it will not be detected
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event",
voltage = 3)
# Check at the end of the test to make sure this event never triggered a Detectable!
# Send Event that is detected
# publish an event for each notification to generate the emails
# since the voltage field in this event is greater than 5, it WILL be detected
rle_publisher = EventPublisher("ExampleDetectableEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event",
voltage = 10)
#-------------------------------------------------------
# make assertions
#-------------------------------------------------------
msg_tuple = proc1.event_processors[notification_id_2].smtp_client.sentmail.get(timeout=4)
# The first event never triggered an email because the voltage was less than 5, the queue is now empty
self.assertTrue(proc1.event_processors[notification_id_2].smtp_client.sentmail.empty())
self.assertEquals(msg_tuple[1], '*****@*****.**' )
#self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS)
# parse the message body
message = msg_tuple[2]
list_lines = message.split("\n")
message_dict = {}
for line in list_lines:
key_item = line.split(": ")
if key_item[0] == 'Subject':
message_dict['Subject'] = key_item[1] + key_item[2]
else:
try:
message_dict[key_item[0]] = key_item[1]
except IndexError as exc:
# these IndexError exceptions happen only because the message sometimes
# has successive /r/n (i.e. new lines) and therefore,
# the indexing goes out of range. These new lines
# can just be ignored. So we ignore the exceptions here.
pass
#self.assertEquals(message_dict['From'], ION_NOTIFICATION_EMAIL_ADDRESS)
self.assertEquals(message_dict['To'], '*****@*****.**')
self.assertEquals(message_dict['Event'].rstrip('\r'), 'DetectionEvent')
self.assertEquals(message_dict['Originator'].rstrip('\r'), 'DetectionEventProcessor')
self.assertEquals(message_dict['Description'].rstrip('\r'), 'Event was detected by DetectionEventProcessor')
@unittest.skip('interface has changed!')
def test_find_event_types_for_resource(self):
# create a dataset object in the RR to pass into the UNS method
dataset_object = IonObject(RT.DataSet, name="dataset1")
dataset_id, version = self.rrc.create(dataset_object)
# get the list of event types for the dataset
events = self.unsc.find_event_types_for_resource(dataset_id)
log.debug("dataset events = " + str(events))
if not events == ['dataset_supplement_added', 'dataset_change']:
self.fail("failed to return correct list of event types")
# try to pass in an id of a resource that doesn't exist (should fail)
try:
events = self.unsc.find_event_types_for_resource("bogus_id")
self.fail("failed to detect non-existant resource")
except:
pass
@unittest.skip('interface has changed!')
def test_create_two_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
notification_id1 = self.unsc.create_notification(notification_object1, user_id)
# create second notification
notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
notification_id2 = self.unsc.create_notification(notification_object2, user_id)
# read the notifications back and check that they are correct
n1 = self.unsc.read_notification(notification_id1)
if n1.name != notification_object1.name or \
n1.origin_list != notification_object1.origin_list or \
n1.events_list != notification_object1.events_list:
self.fail("notification was not correct")
n2 = self.unsc.read_notification(notification_id2)
if n2.name != notification_object2.name or \
n2.origin_list != notification_object2.origin_list or \
n2.events_list != notification_object2.events_list:
self.fail("notification was not correct")
@unittest.skip('interface has changed!')
def test_delete_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
notification1_id = self.unsc.create_notification(notification_object1, user_id)
# create second notification
notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
notification2_id = self.unsc.create_notification(notification_object2, user_id)
# delete both notifications
self.unsc.delete_notification(notification1_id)
self.unsc.delete_notification(notification2_id)
# check that the notifications are not there
try:
n1 = self.unsc.read_notification(notification1_id)
except:
try:
n2 = self.unsc.read_notification(notification2_id)
except:
return
self.fail("failed to delete notifications")
@unittest.skip('interface has changed!')
def test_find_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
self.unsc.create_notification(notification_object, user_id)
# try to find all notifications for user
notifications = self.unsc.find_notifications_by_user(user_id)
if len(notifications) != 2:
self.fail("failed to find all notifications")
@unittest.skip('interface has changed!')
def test_update_user_notification(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create a notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
notification_id = self.unsc.create_notification(notification_object, user_id)
# read back the notification and change it
notification = self.unsc.read_notification(notification_id)
notification.origin_list = ['Some_Resource_Agent_ID5']
self.unsc.update_notification(notification)
# read back the notification and check that it got changed
notification = self.unsc.read_notification(notification_id)
if notification.origin_list != ['Some_Resource_Agent_ID5']:
self.fail("failed to change notification")
@unittest.skip('interface has changed!')
def test_send_notification_emails(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
self.unsc.create_notification(notification_object, user_id)
# publish an event for each notification to generate the emails
# this can't be easily check in SW so need to check for these at the [email protected] account
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event")
de_publisher = EventPublisher("DataEvent")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event")
gevent.sleep(1)
@unittest.skip('interface has changed!')
def test_find_events(self):
# publish some events for the event repository
rle_publisher = EventPublisher("ResourceLifecycleEvent")
de_publisher = EventPublisher("DataEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event1")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event2")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event3")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event1")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event2")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event3")
# find all events for the originator 'Some_Resource_Agent_ID1'
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1')
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID1':
self.fail("failed to find correct events")
# find all events for the originator 'DataEvent'
events = self.unsc.find_events(type='DataEvent')
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'DataEvent':
self.fail("failed to find correct events")
# find 2 events for the originator 'Some_Resource_Agent_ID1'
events = self.unsc.find_events(origin='Some_Resource_Agent_ID2', limit=2)
if len(events) != 2:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID2':
self.fail("failed to find correct events")
# find all events for the originator 'Some_Resource_Agent_ID1' in reverse time order
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1', descending=True)
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID1':
self.fail("failed to find correct events")
class DiscoveryIntTest(IonIntegrationTestCase):
def setUp(self):
super(DiscoveryIntTest, self).setUp()
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
if use_es:
self.es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
self.es_port = CFG.get_safe('server.elasticsearch.port', '9200')
CFG.server.elasticsearch.shards = 1
CFG.server.elasticsearch.replicas = 0
CFG.server.elasticsearch.river_shards = 1
CFG.server.elasticsearch.river_replicas = 0
self.es = ep.ElasticSearch(
host=self.es_host,
port=self.es_port,
timeout=10,
verbose=True
)
op = DotDict(CFG)
op.op = 'clean_bootstrap'
self.container.spawn_process('index_bootstrap','ion.processes.bootstrap.index_bootstrap','IndexBootStrap', op)
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(
host=es_host,
port=es_port,
timeout=10
)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete,index)
IndexManagementService._es_call(es.index_delete,index)
def poll(self, tries, callback, *args, **kwargs):
'''
Polling wrapper for queries
Elasticsearch may not index and cache the changes right away so we may need
a couple of tries and a little time to go by before the results show.
'''
for i in xrange(tries):
retval = callback(*args, **kwargs)
if retval:
return retval
time.sleep(0.2)
return None
def test_traversal(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results = self.discovery.traverse(dp_id)
results.sort()
correct = [pd_id, transform_id]
correct.sort()
self.assertTrue(results == correct, '%s' % results)
def test_iterative_traversal(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results = self.discovery.iterative_traverse(dp_id)
results.sort()
correct = [transform_id]
self.assertTrue(results == correct)
results = self.discovery.iterative_traverse(dp_id, 1)
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@skipIf(not use_es, 'No ElasticSearch')
def test_view_crud(self):
view_id = self.discovery.create_view('big_view',fields=['name'])
catalog_id = self.discovery.list_catalogs(view_id)[0]
index_ids = self.catalog.list_indexes(catalog_id)
self.assertTrue(len(index_ids))
view = self.discovery.read_view(view_id)
self.assertIsInstance(view,View)
self.assertTrue(view.name == 'big_view')
view.name = 'not_so_big_view'
self.discovery.update_view(view)
view = self.discovery.read_view(view_id)
self.assertTrue(view.name == 'not_so_big_view')
self.discovery.delete_view(view_id)
with self.assertRaises(NotFound):
self.discovery.read_view(view_id)
def test_view_best_match(self):
#---------------------------------------------------------------
# Matches the best catalog available OR creates a new one
#---------------------------------------------------------------
catalog_id = self.catalog.create_catalog('dev', keywords=['name','model'])
view_id = self.discovery.create_view('exact_view', fields=['name','model'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids == [catalog_id])
view_id = self.discovery.create_view('another_view', fields=['name','model'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids == [catalog_id])
view_id = self.discovery.create_view('big_view', fields=['name'])
catalog_ids = self.discovery.list_catalogs(view_id)
self.assertTrue(catalog_ids != [catalog_id])
@skipIf(not use_es, 'No ElasticSearch')
def test_basic_searching(self):
#- - - - - - - - - - - - - - - - -
# set up the fake resources
#- - - - - - - - - - - - - - - - -
instrument_pool = [
InstrumentDevice(name='sonobuoy1', hardware_version='1'),
InstrumentDevice(name='sonobuoy2', hardware_version='2'),
InstrumentDevice(name='sonobuoy3', hardware_version='3')
]
for instrument in instrument_pool:
self.rr.create(instrument)
view_id = self.discovery.create_view('devices', fields=['hardware_version'])
search_string = "search 'hardware_version' is '2' from '%s'"%view_id
results = self.poll(5, self.discovery.parse,search_string)
result = results[0]['_source']
self.assertIsInstance(result, InstrumentDevice)
self.assertTrue(result.name == 'sonobuoy2')
self.assertTrue(result.hardware_version == '2')
@skipIf(not use_es, 'No ElasticSearch')
def test_associative_searching(self):
view_id = self.discovery.create_view('devices', fields=['model'])
site_id,_ = self.rr.create(Site('my_site'))
pd_id, _ = self.rr.create(PlatformDevice('my_device', model='abc123'))
self.rr.create_association(subject=site_id, object=pd_id, predicate=PRED.hasDevice)
search_string = "search 'model' is 'abc*' from '%s' and belongs to '%s'"%(view_id, site_id)
results = self.poll(5, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(pd_id in results)
def test_iterative_associative_searching(self):
#--------------------------------------------------------------------------------
# Tests the ability to limit the iterations
#--------------------------------------------------------------------------------
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
search_string = "belongs to '%s' depth 1" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
correct = [transform_id]
self.assertTrue(results == correct, '%s' % results)
search_string = "belongs to '%s' depth 2" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
@skipIf(not use_es, 'No ElasticSearch')
def test_ranged_value_searching(self):
discovery = self.discovery
rr = self.rr
view_id = discovery.create_view('bank_view', fields=['cash_balance'])
bank_id, _ = rr.create(BankAccount(name='broke', cash_balance=10))
search_string = "search 'cash_balance' values from 0 to 100 from '%s'" % view_id
results = self.poll(5, discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == bank_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_collections_searching(self):
site_id, _ = self.rr.create(Site(name='black_mesa'))
view_id = self.discovery.create_view('big', fields=['name'])
# Add the site to a new collection
collection_id = self.ims.create_collection('resource_collection', [site_id])
search_string = "search 'name' is '*' from '%s' and in '%s'" %(view_id, collection_id)
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0] == site_id, '%s' % results)
@skipIf(not use_es, 'No ElasticSearch')
def test_search_by_name(self):
inst_dev = InstrumentDevice(name='test_dev',serial_number='ABC123')
dev_id, _ = self.rr.create(inst_dev)
self.discovery.create_view('devs',fields=['name','serial_number'])
search_string = "search 'serial_number' is 'abc*' from 'devs'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dev_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_search_by_name_index(self):
inst_dev = InstrumentDevice(name='test_dev',serial_number='ABC123')
dev_id, _ = self.rr.create(inst_dev)
search_string = "search 'serial_number' is 'abc*' from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dev_id)
bank_acc = BankAccount(name='blah', cash_balance=10)
res_id , _ = self.rr.create(bank_acc)
search_string = "search 'cash_balance' values from 0 to 100 from 'resources_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == res_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_data_product_search(self):
# Create the dataproduct
dp = DataProduct(name='test_product')
dp.data_format.name = 'test_signal'
dp.data_format.description = 'test signal'
dp.data_format.character_set = 'utf8'
dp.data_format.nominal_sampling_rate_maximum = '44000'
dp.data_format.nominal_sampling_rate_minimum = '44000'
dp.data_product_level = 'basic'
dp_id, _ = self.rr.create(dp)
search_string = "search 'data_format.name' is 'test_signal' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'data_product_level' is 'basic' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
search_string = "search 'data_format.character_set' is 'utf8' from 'data_products_index'"
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_events_search(self):
# Create a resource to force a new event
dp = DataProcess()
dp_id, rev = self.rr.create(dp)
search_string = "SEARCH 'origin' IS '%s' FROM 'events_index'" % dp_id
results = self.poll(9, self.discovery.parse,search_string)
origin_type = results[0]['_source'].origin_type
origin_id = results[0]['_source'].origin
self.assertTrue(origin_type == RT.DataProcess)
self.assertTrue(origin_id == dp_id)
@skipIf(not use_es, 'No ElasticSearch')
def test_geo_distance_search(self):
pd = PlatformDevice(name='test_dev')
pd_id, _ = self.rr.create(pd)
search_string = "search 'nominal_location' geo distance 20 km from lat 0 lon 0 from 'devices_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == pd_id)
self.assertTrue(results[0]['_source'].name == 'test_dev')
@skipIf(not use_es, 'No ElasticSearch')
def test_geo_bbox_search(self):
pd = PlatformDevice(name='test_dev')
pd.nominal_location.lat = 5
pd.nominal_location.lon = 5
pd_id, _ = self.rr.create(pd)
search_string = "search 'nominal_location' geo box top-left lat 10 lon 0 bottom-right lat 0 lon 10 from 'devices_index'"
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == pd_id)
self.assertTrue(results[0]['_source'].name == 'test_dev')
@skipIf(not use_es, 'No ElasticSearch')
def test_user_search(self):
user = UserInfo()
user.name = 'test'
user.contact.phone = '5551212'
user_id, _ = self.rr.create(user)
search_string = 'search "name" is "test" from "users_index"'
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == user_id)
self.assertTrue(results[0]['_source'].name == 'test')
search_string = 'search "contact.phone" is "5551212" from "users_index"'
results = self.poll(9, self.discovery.parse,search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == user_id)
self.assertTrue(results[0]['_source'].name == 'test')
class IndexManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.ims_cli = IndexManagementServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
self.index_name = 'test_index'
def test_create_elasticsearch_index(self):
index_name = self.index_name
ims_cli = self.ims_cli
rr_cli = self.rr_cli
options = SearchOptions()
options.attribute_match = ['test_field']
index_id = ims_cli.create_index(
name=index_name,
content_type=IndexManagementService.ELASTICSEARCH_INDEX,
options=options
)
index_result = self.rr_cli.read(index_id)
self.assertIsInstance(index_result,ElasticSearchIndex)
self.assertTrue(index_result.name == index_name)
#======================================
# Clean up
#======================================
rr_cli.delete(index_id)
def test_create_couchdb_index(self):
index_name = self.index_name
ims_cli = self.ims_cli
rr_cli = self.rr_cli
options = SearchOptions()
options.attribute_match = ['name']
index_id = ims_cli.create_index(
index_name,
content_type=IndexManagementService.COUCHDB_INDEX,
options=options,
datastore_name='fake',
view_name='fake/by_fake'
)
index_result = self.rr_cli.read(index_id)
self.assertIsInstance(index_result,CouchDBIndex)
self.assertTrue(index_result.name==index_name)
#======================================
# Clean up
#======================================
rr_cli.delete(index_id)
def test_read_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
index = ims_cli.read_index(index_id)
self.assertIsInstance(index,Index)
self.assertTrue(index.name==index_name)
rr_cli.delete(index_id)
def test_delete_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
ims_cli.delete_index(index_id)
with self.assertRaises(NotFound):
rr_cli.delete(index_id)
def test_update_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
index = ims_cli.read_index(index_id)
index.name = 'another'
ims_cli.update_index(index)
index = rr_cli.read(index_id)
self.assertTrue(index.name == 'another')
def test_find_indexes(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
#======================================
# Index Pool
#======================================
indexes = [
Index(name='first'),
Index(name='second'),
Index(name='third')
]
id_pool = list()
for index in indexes:
id_pool.append(rr_cli.create(index)[0])
index_id = ims_cli.find_indexes(index_name='second')
index = ims_cli.read_index(index_id)
self.assertTrue(index.name=='second')
#======================================
# Clean up
#======================================
for index_id in id_pool:
rr_cli.delete(index_id)
def test_create_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
with self.assertRaises(BadRequest):
ims_cli.create_collection('failing_collection')
resources = [ Resource(), Resource(), Resource() ]
resources = [ rr_cli.create(i)[0] for i in resources ]
collection_id = ims_cli.create_collection('working_collection',resources)
collection = rr_cli.read(collection_id)
collection_resources = ims_cli.list_collection_resources(collection_id, id_only=True)
self.assertTrue(set(collection_resources) == set(resources), '%s != %s' % (set(collection_resources) , set(resources)))
def test_read_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
collection = Collection(name='working_collection')
collection_id, _ = rr_cli.create(collection)
collection = ims_cli.read_collection(collection_id)
self.assertTrue(collection.name == 'working_collection')
def test_update_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
collection = Collection(name='useful_collection')
collection_id, _ = rr_cli.create(collection)
collection = rr_cli.read(collection_id)
collection.name = 'nub'
ims_cli.update_collection(collection)
collection = rr_cli.read(collection_id)
self.assertTrue(collection.name=='nub')
def test_delete_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
res = Resource()
res_id, rev = rr_cli.create(res)
collection_id = ims_cli.create_collection(name='test_collection', resources=[res_id])
ims_cli.delete_collection(collection_id)
with self.assertRaises(NotFound):
rr_cli.read(collection_id)
def test_list_collection_resources(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
#========================================
# Resource Pool
#========================================
resources = [ InformationResource(name='bean_counter'), InformationResource(name='lunar_rock'), InformationResource('aperature'), InformationResource('lemons') ]
resources = [ rr_cli.create(i)[0] for i in resources ]
collection = Collection(name='park_bench')
collection_id = ims_cli.create_collection(name='park_bench', resources=resources)
retval = ims_cli.list_collection_resources(collection_id, id_only=True)
retval.sort()
resources.sort()
self.assertTrue(retval == resources, '%s != %s' %(retval , resources))
def test_find_collection(self):
res_id, _ = self.rr_cli.create(Resource(name='test_res'))
collection_id = self.ims_cli.create_collection('test', [res_id])
retval = self.ims_cli.find_collection(collection_name='test')
self.assertTrue(retval[0] == collection_id)
retval = self.ims_cli.find_collection(resource_ids=[res_id])
self.assertTrue(retval[0] == collection_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
unittest # suppress an pycharm inspector error if all unittest.skip references are commented out
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode as it depends on modifying CFG on service side')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
self.patch_cfg(CFG["container"], {"extended_resources": {"strip_results": False}})
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
instrument_site_id, _ = self.RR.create(any_old(RT.InstrumentSite))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_site_id, _ = self.RR.create(any_old(RT.PlatformSite))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id)
self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id)
self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
sensor_model_id #is only a target
#create a parsed product for this instrument output
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical')
pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id)
# Lifecycle
self.assertEquals(len(extended_instrument.lcstate_transitions), 6)
self.assertEquals(set(extended_instrument.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate', 'delete']))
self.assertEquals(len(extended_instrument.availability_transitions), 2)
self.assertEquals(set(extended_instrument.availability_transitions.keys()), set(['enable', 'announce']))
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
log.debug("extended_instrument.computed: %s", extended_instrument.computed)
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name)
#check agent instance
inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id)
self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
#compound assoc return list of lists so check the first element
self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name)
extended_platform = self.IMS.get_platform_device_extension(platform_device_id)
self.assertEqual(1, len(extended_platform.portals))
self.assertEqual(1, len(extended_platform.portal_instruments))
#self.assertEqual(1, len(extended_platform.computed.portal_status.value)) # no agent started so NO statuses reported
self.assertEqual(1, len(extended_platform.instrument_devices))
self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id)
self.assertEqual(1, len(extended_platform.instrument_models))
self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id)
self.assertEquals(extended_platform.platform_agent._id, platform_agent_id)
self.assertEquals(len(extended_platform.lcstate_transitions), 6)
self.assertEquals(set(extended_platform.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate', 'delete']))
self.assertEquals(len(extended_platform.availability_transitions), 2)
self.assertEquals(set(extended_platform.availability_transitions.keys()), set(['enable', 'announce']))
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
##check data_product_parameters_set
# !!! OOIION-1342 The UI does not use data_product_parameters_set and it is an expensive calc so the attribute calc was disabled
# !!! Remove check in this test
#self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.data_product_parameters_set.status)
#self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value)
## the ctd parameters should include 'temp'
#self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical'])
#none of these will work because there is no agent
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.firmware_version.status)
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.operational_state.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.power_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.communications_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.data_status_roll_up.status)
# self.assertEqual(DeviceStatusType.STATUS_OK,
# extended_instrument.computed.data_status_roll_up.value)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.location_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.recent_events.status)
# self.assertEqual([], extended_instrument.computed.recent_events.value)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.RR2.pluck(sensor_device_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.OMS.force_delete_instrument_site(instrument_site_id)
self.OMS.force_delete_platform_site(platform_site_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
def test_custom_attributes(self):
"""
Test assignment of custom attributes
"""
instModel_obj = IonObject(OT.CustomAttribute,
name='SBE37IMModelAttr',
description="model custom attr")
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel,
{"custom_attributes":
[instModel_obj]
}))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice,
{"custom_attributes":
{"favorite_color": "red",
"bogus_attr": "should raise warning"
}
}))
self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
# cleanup
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
def _get_datastore(self, dataset_id):
dataset = self.DSC.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_data_producer(self):
idevice_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice))
self.assertEqual(1, len(self.RR2.find_data_producer_ids_of_instrument_device_using_has_data_producer(idevice_id)))
pdevice_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
self.assertEqual(1, len(self.RR2.find_data_producer_ids_of_platform_device_using_has_data_producer(pdevice_id)))
@attr('PREP')
def test_prepare_resource_support(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
instrument_device2_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
instrument_device3_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_device2_id, _ = self.RR.create(any_old(RT.PlatformDevice))
sensor_device2_id, _ = self.RR.create(any_old(RT.SensorDevice))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device_id)
self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device2_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
self.RR.create_association(platform_device2_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device2_id, PRED.hasDevice, instrument_device2_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
self.RR.create_association(sensor_device2_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device2_id, PRED.hasDevice, instrument_device2_id)
sensor_model_id #is only a target
#set lcstate - used for testing prepare - not setting all to DEVELOP, only some
self.RR.execute_lifecycle_transition(instrument_agent_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(instrument_device_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(instrument_device2_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(platform_device_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(platform_device2_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(platform_agent_id, LCE.DEVELOP)
#create a parsed product for this instrument output
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical')
pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
def ion_object_encoder(obj):
return obj.__dict__
#First call to create
instrument_data = self.IMS.prepare_instrument_device_support()
#print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_data._id, '')
self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport)
self.assertEqual(len(instrument_data.associations['InstrumentModel'].resources), 1)
self.assertEqual(instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id)
self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].resources), 1)
self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].resources[0]._id, instrument_agent_instance_id)
self.assertEqual(len(instrument_data.associations['InstrumentModel'].associated_resources), 0)
self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].associated_resources), 0)
self.assertEqual(len(instrument_data.associations['SensorDevice'].resources), 0)
#Next call to update
instrument_data = self.IMS.prepare_instrument_device_support(instrument_device_id)
#print 'Update results'
#print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_data._id, instrument_device_id)
self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport)
self.assertEqual(len(instrument_data.associations['InstrumentModel'].resources), 1)
self.assertEqual(instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id)
self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].resources), 1)
self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].resources[0]._id, instrument_agent_instance_id)
self.assertEqual(len(instrument_data.associations['InstrumentModel'].associated_resources), 1)
self.assertEqual(instrument_data.associations['InstrumentModel'].associated_resources[0].s, instrument_device_id)
self.assertEqual(instrument_data.associations['InstrumentModel'].associated_resources[0].o, instrument_model_id)
self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].associated_resources), 1)
self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].associated_resources[0].o, instrument_agent_instance_id)
self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].associated_resources[0].s, instrument_device_id)
self.assertEqual(len(instrument_data.associations['SensorDevice'].resources), 1)
self.assertEqual(instrument_data.associations['SensorDevice'].resources[0]._id, sensor_device_id)
self.assertEqual(len(instrument_data.associations['SensorDevice'].associated_resources), 1)
self.assertEqual(instrument_data.associations['SensorDevice'].associated_resources[0].o, instrument_device_id)
self.assertEqual(instrument_data.associations['SensorDevice'].associated_resources[0].s, sensor_device_id)
self.assertEqual(instrument_data.associations['InstrumentModel'].assign_request.request_parameters['instrument_device_id'], instrument_device_id)
#test prepare for create of instrument agent instance
instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support()
#print 'Update results'
#print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_agent_data._id, '')
self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport)
self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].resources), 2)
self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].resources), 1)
self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].resources[0]._id, instrument_agent_id)
self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].associated_resources), 0)
self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].associated_resources), 0)
#test prepare for update of instrument agent instance to see if it is associated with the instrument that was created
instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support(instrument_agent_instance_id=instrument_agent_instance_id)
#print 'Update results'
#print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_agent_data._id, instrument_agent_instance_id)
self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport)
self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].resources), 3)
self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].resources), 1)
self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].resources[0]._id, instrument_agent_id)
self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].associated_resources), 1)
self.assertEqual(instrument_agent_data.associations['InstrumentDevice'].associated_resources[0].s, instrument_device_id)
self.assertEqual(instrument_agent_data.associations['InstrumentDevice'].associated_resources[0].o, instrument_agent_instance_id)
self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].associated_resources), 1)
self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].associated_resources[0].o, instrument_agent_id)
self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].associated_resources[0].s, instrument_agent_instance_id)
self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].assign_request.request_parameters['instrument_agent_instance_id'], instrument_agent_instance_id)
#test prepare for update of data product to see if it is associated with the instrument that was created
data_product_data = self.DP.prepare_data_product_support(data_product_id1)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent=2)
self.assertEqual(data_product_data._id, data_product_id1)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 1)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1)
self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, data_product_id1)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0)
self.assertEqual(len(data_product_data.associations['InstrumentDeviceHasOutputProduct'].resources), 3)
self.assertEqual(len(data_product_data.associations['InstrumentDeviceHasOutputProduct'].associated_resources), 1)
self.assertEqual(data_product_data.associations['InstrumentDeviceHasOutputProduct'].associated_resources[0].s, instrument_device_id)
self.assertEqual(data_product_data.associations['InstrumentDeviceHasOutputProduct'].associated_resources[0].o, data_product_id1)
self.assertEqual(len(data_product_data.associations['PlatformDevice'].resources), 2)
platform_data = self.IMS.prepare_platform_device_support()
#print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2)
self.assertEqual(platform_data._id, '')
self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport)
self.assertEqual(len(platform_data.associations['PlatformModel'].resources), 1)
self.assertEqual(platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id)
self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].resources), 1)
self.assertEqual(platform_data.associations['PlatformAgentInstance'].resources[0]._id, platform_agent_instance_id)
self.assertEqual(len(platform_data.associations['PlatformModel'].associated_resources), 0)
self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].associated_resources), 0)
self.assertEqual(len(platform_data.associations['InstrumentDevice'].resources), 1)
platform_data = self.IMS.prepare_platform_device_support(platform_device_id)
#print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2)
self.assertEqual(platform_data._id, platform_device_id)
self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport)
self.assertEqual(len(platform_data.associations['PlatformModel'].resources), 1)
self.assertEqual(platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id)
self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].resources), 1)
self.assertEqual(platform_data.associations['PlatformAgentInstance'].resources[0]._id, platform_agent_instance_id)
self.assertEqual(len(platform_data.associations['PlatformModel'].associated_resources), 1)
self.assertEqual(platform_data.associations['PlatformModel'].associated_resources[0].s, platform_device_id)
self.assertEqual(platform_data.associations['PlatformModel'].associated_resources[0].o, platform_model_id)
self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].associated_resources), 1)
self.assertEqual(platform_data.associations['PlatformAgentInstance'].associated_resources[0].o, platform_agent_instance_id)
self.assertEqual(platform_data.associations['PlatformAgentInstance'].associated_resources[0].s, platform_device_id)
self.assertEqual(len(platform_data.associations['InstrumentDevice'].resources), 2)
#self.assertEqual(len(platform_data.associations['InstrumentDevice'].associated_resources), 1)
#self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].s, platform_device_id)
#self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].o, instrument_device_id)
self.assertEqual(platform_data.associations['PlatformModel'].assign_request.request_parameters['platform_device_id'], platform_device_id)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.RR2.pluck(sensor_device_id)
self.RR2.pluck(sensor_device2_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_device(instrument_device2_id)
self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_device(platform_device2_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_device(sensor_device2_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
class TestObservatoryManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.org_management_service = OrgManagementServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#print 'TestObservatoryManagementServiceIntegration: started services'
self.event_publisher = EventPublisher()
# @unittest.skip('this exists only for debugging the launch process')
# def test_just_the_setup(self):
# return
def destroy(self, resource_ids):
self.OMS.force_delete_observatory(resource_ids.observatory_id)
self.OMS.force_delete_subsite(resource_ids.subsite_id)
self.OMS.force_delete_subsite(resource_ids.subsite2_id)
self.OMS.force_delete_subsite(resource_ids.subsiteb_id)
self.OMS.force_delete_subsite(resource_ids.subsitez_id)
self.OMS.force_delete_platform_site(resource_ids.platform_site_id)
self.OMS.force_delete_platform_site(resource_ids.platform_siteb_id)
self.OMS.force_delete_platform_site(resource_ids.platform_siteb2_id)
self.OMS.force_delete_platform_site(resource_ids.platform_site3_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_site_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_site2_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_siteb3_id)
self.OMS.force_delete_instrument_site(resource_ids.instrument_site4_id)
#@unittest.skip('targeting')
def test_observatory_management(self):
resources = self._make_associations()
self._do_test_find_related_sites(resources)
self._do_test_get_sites_devices_status(resources)
self._do_test_find_site_data_products(resources)
self._do_test_find_related_frames_of_reference(resources)
self._do_test_create_geospatial_point_center(resources)
self._do_test_find_observatory_org(resources)
self.destroy(resources)
def _do_test_find_related_sites(self, resources):
site_resources, site_children = self.OMS.find_related_sites(resources.org_id)
#import sys, pprint
#print >> sys.stderr, pprint.pformat(site_resources)
#print >> sys.stderr, pprint.pformat(site_children)
#self.assertIn(resources.org_id, site_resources)
self.assertIn(resources.observatory_id, site_resources)
self.assertIn(resources.subsite_id, site_resources)
self.assertIn(resources.subsite_id, site_resources)
self.assertIn(resources.subsite2_id, site_resources)
self.assertIn(resources.platform_site_id, site_resources)
self.assertIn(resources.instrument_site_id, site_resources)
self.assertEquals(len(site_resources), 13)
self.assertEquals(site_resources[resources.observatory_id].type_, RT.Observatory)
self.assertIn(resources.org_id, site_children)
self.assertIn(resources.observatory_id, site_children)
self.assertIn(resources.subsite_id, site_children)
self.assertIn(resources.subsite_id, site_children)
self.assertIn(resources.subsite2_id, site_children)
self.assertIn(resources.platform_site_id, site_children)
self.assertNotIn(resources.instrument_site_id, site_children)
self.assertEquals(len(site_children), 9)
self.assertIsInstance(site_children[resources.subsite_id], list)
self.assertEquals(len(site_children[resources.subsite_id]), 2)
def _do_test_get_sites_devices_status(self, resources):
result_dict = self.OMS.get_sites_devices_status(resources.org_id)
site_resources = result_dict.get("site_resources", None)
site_children = result_dict.get("site_children", None)
self.assertEquals(len(site_resources), 14)
self.assertEquals(len(site_children), 9)
result_dict = self.OMS.get_sites_devices_status(resources.org_id, include_devices=True, include_status=True)
log.debug("RESULT DICT: %s", result_dict.keys())
site_resources = result_dict.get("site_resources", None)
site_children = result_dict.get("site_children", None)
site_status = result_dict.get("site_status", None)
self.assertEquals(len(site_resources), 14)
self.assertEquals(len(site_children), 9)
result_dict = self.OMS.get_sites_devices_status(resources.observatory_id, include_devices=True, include_status=True)
site_resources = result_dict.get("site_resources")
site_children = result_dict.get("site_children")
site_status = result_dict.get("site_status")
self.assertEquals(len(site_resources), 13)
self.assertEquals(len(site_children), 8)
def _do_test_find_site_data_products(self, resources):
res_dict = self.OMS.find_site_data_products(resources.org_id)
#import sys, pprint
#print >> sys.stderr, pprint.pformat(res_dict)
self.assertIsNone(res_dict['data_product_resources'])
self.assertIn(resources.platform_device_id, res_dict['device_data_products'])
self.assertIn(resources.instrument_device_id, res_dict['device_data_products'])
#@unittest.skip('targeting')
def _do_test_find_related_frames_of_reference(self, stuff):
# finding subordinates gives a dict of obj lists, convert objs to ids
def idify(adict):
ids = {}
for k, v in adict.iteritems():
ids[k] = []
for obj in v:
ids[k].append(obj._id)
return ids
# a short version of the function we're testing, with id-ify
def short(resource_id, output_types):
ret = self.OMS.find_related_frames_of_reference(resource_id,
output_types)
return idify(ret)
#set up associations first
stuff = self._make_associations()
#basic traversal of tree from instrument to platform
ids = short(stuff.instrument_site_id, [RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
#since this is the first search, just make sure the input inst_id got stripped
if RT.InstrumentSite in ids:
self.assertNotIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
#basic traversal of tree from platform to instrument
ids = short(stuff.platform_siteb_id, [RT.InstrumentSite])
self.assertIn(RT.InstrumentSite, ids)
self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
self.assertNotIn(stuff.instrument_site2_id, ids[RT.InstrumentSite])
#full traversal of tree from observatory down to instrument
ids = short(stuff.observatory_id, [RT.InstrumentSite])
self.assertIn(RT.InstrumentSite, ids)
self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
#full traversal of tree from instrument to observatory
ids = short(stuff.instrument_site_id, [RT.Observatory])
self.assertIn(RT.Observatory, ids)
self.assertIn(stuff.observatory_id, ids[RT.Observatory])
#partial traversal, only down to platform
ids = short(stuff.observatory_id, [RT.Subsite, RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(RT.Subsite, ids)
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_site3_id, ids[RT.PlatformSite])
self.assertIn(stuff.subsite_id, ids[RT.Subsite])
self.assertIn(stuff.subsite2_id, ids[RT.Subsite])
self.assertIn(stuff.subsitez_id, ids[RT.Subsite])
self.assertIn(stuff.subsiteb_id, ids[RT.Subsite])
self.assertNotIn(RT.InstrumentSite, ids)
#partial traversal, only down to platform
ids = short(stuff.instrument_site_id, [RT.Subsite, RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(RT.Subsite, ids)
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.subsite_id, ids[RT.Subsite])
self.assertIn(stuff.subsiteb_id, ids[RT.Subsite])
self.assertNotIn(stuff.subsite2_id, ids[RT.Subsite])
self.assertNotIn(stuff.subsitez_id, ids[RT.Subsite])
self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
self.assertNotIn(RT.Observatory, ids)
self.destroy(stuff)
def _make_associations(self):
"""
create one of each resource and association used by OMS
to guard against problems in ion-definitions
"""
#raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41")
"""
the tree we're creating (observatory, sites, platforms, instruments)
rows are lettered, colums numbered.
- first row is implied a
- first column is implied 1
- site Z, just because
O--Sz
|
S--S2--P3--I4
|
Sb-Pb2-Ib3
|
P--I2 <- PlatformDevice, InstrumentDevice2
|
Pb <- PlatformDevice b
|
I <- InstrumentDevice
"""
org_id = self.OMS.create_marine_facility(any_old(RT.Org))
def create_under_org(resource_type, extra_fields=None):
obj = any_old(resource_type, extra_fields)
if RT.InstrumentDevice == resource_type:
resource_id = self.IMS.create_instrument_device(obj)
else:
resource_id, _ = self.RR.create(obj)
self.OMS.assign_resource_to_observatory_org(resource_id=resource_id, org_id=org_id)
return resource_id
#stuff we control
observatory_id = create_under_org(RT.Observatory)
subsite_id = create_under_org(RT.Subsite)
subsite2_id = create_under_org(RT.Subsite)
subsiteb_id = create_under_org(RT.Subsite)
subsitez_id = create_under_org(RT.Subsite)
platform_site_id = create_under_org(RT.PlatformSite)
platform_siteb_id = create_under_org(RT.PlatformSite)
platform_siteb2_id = create_under_org(RT.PlatformSite)
platform_site3_id = create_under_org(RT.PlatformSite)
instrument_site_id = create_under_org(RT.InstrumentSite)
instrument_site2_id = create_under_org(RT.InstrumentSite)
instrument_siteb3_id = create_under_org(RT.InstrumentSite)
instrument_site4_id = create_under_org(RT.InstrumentSite)
#stuff we associate to
instrument_device_id = create_under_org(RT.InstrumentDevice)
instrument_device2_id = create_under_org(RT.InstrumentDevice)
platform_device_id = create_under_org(RT.PlatformDevice)
platform_deviceb_id = create_under_org(RT.PlatformDevice)
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
deployment_id, _ = self.RR.create(any_old(RT.Deployment))
#observatory
self.RR.create_association(observatory_id, PRED.hasSite, subsite_id)
self.RR.create_association(observatory_id, PRED.hasSite, subsitez_id)
#site
self.RR.create_association(subsite_id, PRED.hasSite, subsite2_id)
self.RR.create_association(subsite_id, PRED.hasSite, subsiteb_id)
self.RR.create_association(subsite2_id, PRED.hasSite, platform_site3_id)
self.RR.create_association(subsiteb_id, PRED.hasSite, platform_siteb2_id)
self.RR.create_association(subsiteb_id, PRED.hasSite, platform_site_id)
#platform_site(s)
self.RR.create_association(platform_site3_id, PRED.hasSite, instrument_site4_id)
self.RR.create_association(platform_siteb2_id, PRED.hasSite, instrument_siteb3_id)
self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site2_id)
self.RR.create_association(platform_site_id, PRED.hasSite, platform_siteb_id)
self.RR.create_association(platform_siteb_id, PRED.hasSite, instrument_site_id)
self.RR.create_association(platform_siteb_id, PRED.hasDevice, platform_deviceb_id)
#test network parent link
self.OMS.assign_device_to_network_parent(platform_device_id, platform_deviceb_id)
self.RR.create_association(platform_site_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id)
self.RR.create_association(platform_site_id, PRED.hasDeployment, deployment_id)
#instrument_site(s)
self.RR.create_association(instrument_site_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id)
self.RR.create_association(instrument_site_id, PRED.hasDeployment, deployment_id)
self.RR.create_association(instrument_site2_id, PRED.hasDevice, instrument_device2_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id)
ret = DotDict()
ret.org_id = org_id
ret.observatory_id = observatory_id
ret.subsite_id = subsite_id
ret.subsite2_id = subsite2_id
ret.subsiteb_id = subsiteb_id
ret.subsitez_id = subsitez_id
ret.platform_site_id = platform_site_id
ret.platform_siteb_id = platform_siteb_id
ret.platform_siteb2_id = platform_siteb2_id
ret.platform_site3_id = platform_site3_id
ret.instrument_site_id = instrument_site_id
ret.instrument_site2_id = instrument_site2_id
ret.instrument_siteb3_id = instrument_siteb3_id
ret.instrument_site4_id = instrument_site4_id
ret.instrument_device_id = instrument_device_id
ret.instrument_device2_id = instrument_device2_id
ret.platform_device_id = platform_device_id
ret.platform_deviceb_id = platform_deviceb_id
ret.instrument_model_id = instrument_model_id
ret.platform_model_id = platform_model_id
ret.deployment_id = deployment_id
return ret
#@unittest.skip("targeting")
def test_create_observatory(self):
observatory_obj = IonObject(RT.Observatory,
name='TestFacility',
description='some new mf')
observatory_id = self.OMS.create_observatory(observatory_obj)
self.OMS.force_delete_observatory(observatory_id)
#@unittest.skip("targeting")
def _do_test_create_geospatial_point_center(self, resources):
platformsite_obj = IonObject(RT.PlatformSite,
name='TestPlatformSite',
description='some new TestPlatformSite')
geo_index_obj = IonObject(OT.GeospatialBounds)
geo_index_obj.geospatial_latitude_limit_north = 20.0
geo_index_obj.geospatial_latitude_limit_south = 10.0
geo_index_obj.geospatial_longitude_limit_east = 15.0
geo_index_obj.geospatial_longitude_limit_west = 20.0
platformsite_obj.constraint_list = [geo_index_obj]
platformsite_id = self.OMS.create_platform_site(platformsite_obj)
# now get the dp back to see if it was updated
platformsite_obj = self.OMS.read_platform_site(platformsite_id)
self.assertEquals('some new TestPlatformSite', platformsite_obj.description)
self.assertAlmostEqual(15.0, platformsite_obj.geospatial_point_center.lat, places=1)
#now adjust a few params
platformsite_obj.description = 'some old TestPlatformSite'
geo_index_obj = IonObject(OT.GeospatialBounds)
geo_index_obj.geospatial_latitude_limit_north = 30.0
geo_index_obj.geospatial_latitude_limit_south = 20.0
platformsite_obj.constraint_list = [geo_index_obj]
update_result = self.OMS.update_platform_site(platformsite_obj)
# now get the dp back to see if it was updated
platformsite_obj = self.OMS.read_platform_site(platformsite_id)
self.assertEquals('some old TestPlatformSite', platformsite_obj.description)
self.assertAlmostEqual(25.0, platformsite_obj.geospatial_point_center.lat, places=1)
self.OMS.force_delete_platform_site(platformsite_id)
#@unittest.skip("targeting")
def _do_test_find_observatory_org(self, resources):
log.debug("Make TestOrg")
org_obj = IonObject(RT.Org,
name='TestOrg',
description='some new mf org')
org_id = self.OMS.create_marine_facility(org_obj)
log.debug("Make Observatory")
observatory_obj = IonObject(RT.Observatory,
name='TestObservatory',
description='some new obs')
observatory_id = self.OMS.create_observatory(observatory_obj)
log.debug("assign observatory to org")
self.OMS.assign_resource_to_observatory_org(observatory_id, org_id)
log.debug("verify assigment")
org_objs = self.OMS.find_org_by_observatory(observatory_id)
self.assertEqual(1, len(org_objs))
self.assertEqual(org_id, org_objs[0]._id)
log.debug("org_id=<" + org_id + ">")
log.debug("create a subsite with parent Observatory")
subsite_obj = IonObject(RT.Subsite,
name= 'TestSubsite',
description = 'sample subsite')
subsite_id = self.OMS.create_subsite(subsite_obj, observatory_id)
self.assertIsNotNone(subsite_id, "Subsite not created.")
log.debug("verify that Subsite is linked to Observatory")
mf_subsite_assoc = self.RR.get_association(observatory_id, PRED.hasSite, subsite_id)
self.assertIsNotNone(mf_subsite_assoc, "Subsite not connected to Observatory.")
log.debug("add the Subsite as a resource of this Observatory")
self.OMS.assign_resource_to_observatory_org(resource_id=subsite_id, org_id=org_id)
log.debug("verify that Subsite is linked to Org")
org_subsite_assoc = self.RR.get_association(org_id, PRED.hasResource, subsite_id)
self.assertIsNotNone(org_subsite_assoc, "Subsite not connected as resource to Org.")
log.debug("create a logical platform with parent Subsite")
platform_site_obj = IonObject(RT.PlatformSite,
name= 'TestPlatformSite',
description = 'sample logical platform')
platform_site_id = self.OMS.create_platform_site(platform_site_obj, subsite_id)
self.assertIsNotNone(platform_site_id, "PlatformSite not created.")
log.debug("verify that PlatformSite is linked to Site")
site_lp_assoc = self.RR.get_association(subsite_id, PRED.hasSite, platform_site_id)
self.assertIsNotNone(site_lp_assoc, "PlatformSite not connected to Site.")
log.debug("add the PlatformSite as a resource of this Observatory")
self.OMS.assign_resource_to_observatory_org(resource_id=platform_site_id, org_id=org_id)
log.debug("verify that PlatformSite is linked to Org")
org_lp_assoc = self.RR.get_association(org_id, PRED.hasResource, platform_site_id)
self.assertIsNotNone(org_lp_assoc, "PlatformSite not connected as resource to Org.")
log.debug("create a logical instrument with parent logical platform")
instrument_site_obj = IonObject(RT.InstrumentSite,
name= 'TestInstrumentSite',
description = 'sample logical instrument')
instrument_site_id = self.OMS.create_instrument_site(instrument_site_obj, platform_site_id)
self.assertIsNotNone(instrument_site_id, "InstrumentSite not created.")
log.debug("verify that InstrumentSite is linked to PlatformSite")
li_lp_assoc = self.RR.get_association(platform_site_id, PRED.hasSite, instrument_site_id)
self.assertIsNotNone(li_lp_assoc, "InstrumentSite not connected to PlatformSite.")
log.debug("add the InstrumentSite as a resource of this Observatory")
self.OMS.assign_resource_to_observatory_org(resource_id=instrument_site_id, org_id=org_id)
log.debug("verify that InstrumentSite is linked to Org")
org_li_assoc = self.RR.get_association(org_id, PRED.hasResource, instrument_site_id)
self.assertIsNotNone(org_li_assoc, "InstrumentSite not connected as resource to Org.")
log.debug("remove the InstrumentSite as a resource of this Observatory")
self.OMS.unassign_resource_from_observatory_org(instrument_site_id, org_id)
log.debug("verify that InstrumentSite is linked to Org")
assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.InstrumentSite, id_only=True )
self.assertEqual(0, len(assocs))
log.debug("remove the InstrumentSite, association should drop automatically")
self.OMS.delete_instrument_site(instrument_site_id)
assocs, _ = self.RR.find_objects(platform_site_id, PRED.hasSite, RT.InstrumentSite, id_only=True )
self.assertEqual(0, len(assocs))
log.debug("remove the PlatformSite as a resource of this Observatory")
self.OMS.unassign_resource_from_observatory_org(platform_site_id, org_id)
log.debug("verify that PlatformSite is linked to Org")
assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.PlatformSite, id_only=True )
self.assertEqual(0, len(assocs))
log.debug("remove the Site as a resource of this Observatory")
self.OMS.unassign_resource_from_observatory_org(subsite_id, org_id)
log.debug("verify that Site is linked to Org")
assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.Subsite, id_only=True )
self.assertEqual(0, len(assocs))
self.RR.delete(org_id)
self.OMS.force_delete_observatory(observatory_id)
self.OMS.force_delete_subsite(subsite_id)
self.OMS.force_delete_platform_site(platform_site_id)
self.OMS.force_delete_instrument_site(instrument_site_id)
@attr('EXT')
def test_observatory_extensions(self):
obs_id = self.RR2.create(any_old(RT.Observatory))
pss_id = self.RR2.create(any_old(RT.PlatformSite, dict(alt_resource_type="StationSite")))
pas_id = self.RR2.create(any_old(RT.PlatformSite, dict(alt_resource_type="PlatformAssemblySite")))
pcs_id = self.RR2.create(any_old(RT.PlatformSite, dict(alt_resource_type="PlatformComponentSite")))
ins_id = self.RR2.create(any_old(RT.InstrumentSite))
obs_obj = self.RR2.read(obs_id)
pss_obj = self.RR2.read(pss_id)
pas_obj = self.RR2.read(pas_id)
pcs_obj = self.RR2.read(pcs_id)
ins_obj = self.RR2.read(ins_id)
self.RR2.create_association(obs_id, PRED.hasSite, pss_id)
self.RR2.create_association(pss_id, PRED.hasSite, pas_id)
self.RR2.create_association(pas_id, PRED.hasSite, pcs_id)
self.RR2.create_association(pcs_id, PRED.hasSite, ins_id)
extended_obs = self.OMS.get_observatory_site_extension(obs_id, user_id=12345)
self.assertEqual([pss_obj], extended_obs.computed.platform_station_sites.value)
self.assertEqual(ComputedValueAvailability.PROVIDED, extended_obs.computed.platform_station_sites.status)
self.assertEqual([pas_obj], extended_obs.computed.platform_assembly_sites.value)
self.assertEqual(ComputedValueAvailability.PROVIDED, extended_obs.computed.platform_assembly_sites.status)
self.assertEqual([pcs_obj], extended_obs.computed.platform_component_sites.value)
self.assertEqual(ComputedValueAvailability.PROVIDED, extended_obs.computed.platform_component_sites.status)
self.assertEqual([ins_obj], extended_obs.computed.instrument_sites.value)
extended_pss = self.OMS.get_observatory_site_extension(obs_id, user_id=12345)
self.assertEqual([pas_obj], extended_pss.computed.platform_assembly_sites.value)
self.assertEqual([pcs_obj], extended_pss.computed.platform_component_sites.value)
self.assertEqual([ins_obj], extended_pss.computed.instrument_sites.value)
extended_pas = self.OMS.get_observatory_site_extension(pas_id, user_id=12345)
self.assertEqual([pcs_obj], extended_pas.computed.platform_component_sites.value)
self.assertEqual([ins_obj], extended_pas.computed.instrument_sites.value)
extended_pcs = self.OMS.get_platform_component_site_extension(pcs_id, user_id=12345)
self.assertEqual([ins_obj], extended_pcs.computed.instrument_sites.value)
#@unittest.skip("in development...")
@attr('EXT')
@attr('EXT1')
def test_observatory_org_extended(self):
stuff = self._make_associations()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',
id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed',
parameter_dictionary_id=parsed_pdict_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=stuff.instrument_device_id,
data_product_id=data_product_id1)
#Create a user to be used as regular member
member_actor_obj = IonObject(RT.ActorIdentity, name='org member actor')
member_actor_id,_ = self.RR.create(member_actor_obj)
assert(member_actor_id)
member_actor_header = get_actor_header(member_actor_id)
member_user_obj = IonObject(RT.UserInfo, name='org member user')
member_user_id,_ = self.RR.create(member_user_obj)
assert(member_user_id)
self.RR.create_association(subject=member_actor_id, predicate=PRED.hasInfo, object=member_user_id)
#Build the Service Agreement Proposal to enroll a user actor
sap = IonObject(OT.EnrollmentProposal,consumer=member_actor_id, provider=stuff.org_id )
sap_response = self.org_management_service.negotiate(sap, headers=member_actor_header )
#enroll the member without using negotiation
self.org_management_service.enroll_member(org_id=stuff.org_id, actor_id=member_actor_id)
#--------------------------------------------------------------------------------
# Get the extended Site (platformSite)
#--------------------------------------------------------------------------------
try:
extended_site = self.OMS.get_site_extension(stuff.platform_site_id)
except:
log.error('failed to get extended site', exc_info=True)
raise
log.debug("extended_site: %r ", extended_site)
self.assertEqual(1, len(extended_site.platform_devices))
self.assertEqual(1, len(extended_site.platform_models))
self.assertEqual(stuff.platform_device_id, extended_site.platform_devices[0]._id)
self.assertEqual(stuff.platform_model_id, extended_site.platform_models[0]._id)
log.debug("verify that PlatformDeviceb is linked to PlatformDevice with hasNetworkParent link")
associations = self.RR.find_associations(subject=stuff.platform_deviceb_id, predicate=PRED.hasNetworkParent, object=stuff.platform_device_id, id_only=True)
self.assertIsNotNone(associations, "PlatformDevice child not connected to PlatformDevice parent.")
#--------------------------------------------------------------------------------
# Get the extended Org
#--------------------------------------------------------------------------------
#test the extended resource
extended_org = self.OMS.get_marine_facility_extension(stuff.org_id)
log.debug("test_observatory_org_extended: extended_org: %s ", str(extended_org))
#self.assertEqual(2, len(extended_org.instruments_deployed) )
#self.assertEqual(1, len(extended_org.platforms_not_deployed) )
self.assertEqual(2, extended_org.number_of_platforms)
self.assertEqual(2, len(extended_org.platform_models) )
self.assertEqual(2, extended_org.number_of_instruments)
self.assertEqual(2, len(extended_org.instrument_models) )
self.assertEqual(1, len(extended_org.members))
self.assertNotEqual(extended_org.members[0]._id, member_actor_id)
self.assertEqual(extended_org.members[0]._id, member_user_id)
self.assertEqual(1, len(extended_org.open_requests))
self.assertTrue(len(extended_site.deployments)>0)
self.assertEqual(len(extended_site.deployments), len(extended_site.deployment_info))
#test the extended resource of the ION org
ion_org_id = self.org_management_service.find_org()
extended_org = self.OMS.get_marine_facility_extension(ion_org_id._id, user_id=12345)
log.debug("test_observatory_org_extended: extended_ION_org: %s ", str(extended_org))
self.assertEqual(1, len(extended_org.members))
self.assertEqual(0, extended_org.number_of_platforms)
#self.assertEqual(1, len(extended_org.sites))
#--------------------------------------------------------------------------------
# Get the extended Site
#--------------------------------------------------------------------------------
#create device state events to use for op /non-op filtering in extended
t = get_ion_ts()
self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent',
origin = stuff.instrument_device_id, state=ResourceAgentState.STREAMING )
self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent',
origin = stuff.instrument_device2_id, state=ResourceAgentState.INACTIVE )
extended_site = self.OMS.get_site_extension(stuff.instrument_site2_id)
log.debug("test_observatory_org_extended: extended_site: %s ", str(extended_site))
self.dpclient.delete_data_product(data_product_id1)
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(TestActivateInstrumentIntegration.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(
host=es_host,
port=es_port,
timeout=10
)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete,index)
IndexManagementService._es_call(es.index_delete,index)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_notification(self, user_name = '', instrument_id='', product_id=''):
#--------------------------------------------------------------------------------------
# Make notification request objects
#--------------------------------------------------------------------------------------
notification_request_1 = NotificationRequest( name= 'notification_1',
origin=instrument_id,
origin_type="instrument",
event_type='ResourceLifecycleEvent')
notification_request_2 = NotificationRequest( name='notification_2',
origin=product_id,
origin_type="data product",
event_type='DetectionEvent')
#--------------------------------------------------------------------------------------
# Create a user and get the user_id
#--------------------------------------------------------------------------------------
user = UserInfo()
user.name = user_name
user.contact.email = '*****@*****.**' % user_name
user_id, _ = self.rrclient.create(user)
#--------------------------------------------------------------------------------------
# Create notification
#--------------------------------------------------------------------------------------
self.usernotificationclient.create_notification(notification=notification_request_1, user_id=user_id)
self.usernotificationclient.create_notification(notification=notification_request_2, user_id=user_id)
log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id) )
return user_id
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def _check_computed_attributes_of_extended_instrument(self, expected_instrument_device_id = '',extended_instrument = None):
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
# the following assert will not work without elasticsearch.
#self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
self.assertEqual(extended_instrument.computed.communications_status_roll_up.value, StatusType.STATUS_WARNING)
self.assertEqual(extended_instrument.computed.data_status_roll_up.value, StatusType.STATUS_OK)
self.assertEqual(extended_instrument.computed.power_status_roll_up.value, StatusType.STATUS_WARNING)
# Verify the computed attribute for user notification requests
self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
notifications = extended_instrument.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_instrument_device_id)
self.assertEqual(notification.origin_type, "instrument")
self.assertEqual(notification.event_type, 'ResourceLifecycleEvent')
def _check_computed_attributes_of_extended_product(self, expected_data_product_id = '', extended_data_product = None):
self.assertEqual(expected_data_product_id, extended_data_product._id)
log.debug("extended_data_product.computed: %s", extended_data_product.computed)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_data_product.computed.product_download_size_estimated, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.number_active_subscriptions, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue)
self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.user_notification_requests, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.active_user_subscriptions, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.past_user_subscriptions, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_contents_updated, ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue)
# exact text here keeps changing to fit UI capabilities. keep assertion general...
self.assertTrue( 'ok' in extended_data_product.computed.last_granule.value['quality_flag'] )
self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value) )
notifications = extended_data_product.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_data_product_id)
self.assertEqual(notification.origin_type, "data product")
self.assertEqual(notification.event_type, 'DetectionEvent')
@attr('LOCOINT')
@unittest.skipIf(not use_es, 'No ElasticSearch')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}})
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
#Create stream alarms
"""
test_two_sided_interval
Test interval alarm and alarm event publishing for a closed
inteval.
"""
# kwargs = {
# 'name' : 'test_sim_warning',
# 'stream_name' : 'parsed',
# 'value_id' : 'temp',
# 'message' : 'Temperature is above test range of 5.0.',
# 'type' : StreamAlarmType.WARNING,
# 'upper_bound' : 5.0,
# 'upper_rel_op' : '<'
# }
kwargs = {
'name' : 'temperature_warning_interval',
'stream_name' : 'parsed',
'value_id' : 'temp',
'message' : 'Temperature is below the normal range of 50.0 and above.',
'type' : StreamAlarmType.WARNING,
'lower_bound' : 50.0,
'lower_rel_op' : '<'
}
# Create alarm object.
alarm = {}
alarm['type'] = 'IntervalAlarmDef'
alarm['kwargs'] = kwargs
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5, alarms=[alarm] )
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg",
stream_configurations = [raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) " , instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s' , data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s' , dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0] )
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# setup notifications for the device and parsed data product
user_id_1 = self._create_notification( user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1)
#---------- Create notifications for another user and verify that we see different computed subscriptions for the two users ---------
user_id_2 = self._create_notification( user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug('Data product streams2 = %s' , str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id2 = %s' , dataset_ids[0])
self.raw_dataset = dataset_ids[0]
#elastic search debug
es_indexes, _ = self.container.resource_registry.find_resources(restype='ElasticSearchIndex')
log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes])
log.debug('Bootstrap %s', CFG.bootstrap.use_es)
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#setup a subscriber to alarm events from the device
self._events_received= []
self._event_count = 0
self._samples_out_of_range = 0
self._samples_complete = False
self._async_sample_result = AsyncResult()
def consume_event(*args, **kwargs):
log.debug('TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
self._event_count = len(self._events_received)
self._async_sample_result.set()
self._event_subscriber = EventSubscriber(
event_type= 'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent
callback=consume_event,
origin=instDevice_id)
self._event_subscriber.start()
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
def stop_subscriber():
self._event_subscriber.stop()
self._event_subscriber = None
self.addCleanup(stop_subscriber)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
inst_agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
inst_agent_instance_obj.agent_process_id)
log.debug('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=inst_agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s" , str(self._ia_client))
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s" , str(retval))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s" , str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s" , str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s" , str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
for i in xrange(10):
retval = self._ia_client.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from sample %s" , str(retval))
log.debug( "test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s" , str(reply))
self._samples_complete = True
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt.pretty_print()) )
temp_vals = rdt['temp']
self.assertEquals(len(temp_vals) , 10)
log.debug("test_activateInstrumentSample: all temp_vals: %s", temp_vals )
#out_of_range_temp_vals = [i for i in temp_vals if i > 5]
out_of_range_temp_vals = [i for i in temp_vals if i < 50.0]
log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s", out_of_range_temp_vals )
self._samples_out_of_range = len(out_of_range_temp_vals)
# if no bad values were produced, then do not wait for an event
if self._samples_out_of_range == 0:
self._async_sample_result.set()
log.debug("test_activateInstrumentSample: _events_received: %s", self._events_received )
log.debug("test_activateInstrumentSample: _event_count: %s", self._event_count )
self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout)
replay_data = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("RDT raw: %s", str(rdt.pretty_print()) )
raw_vals = rdt['raw']
self.assertEquals(len(raw_vals) , 10)
log.debug("l4-ci-sa-rq-138")
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
#--------------------------------------------------------------------------------
# Get the extended data product to see if it contains the granules
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id1, user_id=user_id_1)
def poller(extended_product):
return len(extended_product.computed.user_notification_requests.value) == 1
poll(poller, extended_product, timeout=30)
self._check_computed_attributes_of_extended_product( expected_data_product_id = data_product_id1, extended_data_product = extended_product)
#--------------------------------------------------------------------------------
#put some events into the eventsdb to test - this should set the comms and data status to WARNING
#--------------------------------------------------------------------------------
t = get_ion_ts()
self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceStatusEvent',
origin = instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values = [200] )
self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceCommsEvent',
origin = instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds = 20 )
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_1)
self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument)
#--------------------------------------------------------------------------------
# For the second user, check the extended data product and the extended intrument
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id2, user_id=user_id_2)
self._check_computed_attributes_of_extended_product(expected_data_product_id = data_product_id2, extended_data_product = extended_product)
#---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING ---------
t = get_ion_ts()
self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceStatusEvent',
origin = instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values = [200] )
self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceCommsEvent',
origin = instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds = 20 )
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_2)
self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument)
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(data_product_id1)
self.dpclient.delete_data_product(data_product_id2)
class TestTransformWorker(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Instantiate a process to represent the test
process = TransformWorkerTestProcess()
self.dataset_management_client = DatasetManagementServiceClient(
node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceProcessClient(
node=self.container.node, process=process)
self.time_dom, self.spatial_dom = time_series_domain()
self.ph = ParameterHelper(self.dataset_management_client,
self.addCleanup)
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
def push_granule(self, data_product_id):
'''
Publishes and monitors that the granule arrived
'''
datasets, _ = self.rrclient.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)
dataset_monitor = DatasetMonitor(datasets[0])
rdt = self.ph.rdt_for_data_product(data_product_id)
self.ph.fill_parsed_rdt(rdt)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
assert dataset_monitor.wait()
dataset_monitor.stop()
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_transform_worker(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.dp_list = []
self.data_process_objs = []
self._output_stream_ids = []
self.granule_verified = Event()
self.worker_assigned_event_verified = Event()
self.dp_created_event_verified = Event()
self.heartbeat_event_verified = Event()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
self.start_event_listener()
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process(
)
self.dp_list.append(dataprocess_id)
# validate the repository for data product algorithms persists the new resources NEW SA-1
# create_data_process call created one of each
dpd_ids, _ = self.rrclient.find_resources(
restype=OT.DataProcessDefinition, id_only=False)
# there will be more than one becuase of the DPDs that reperesent the PFs in the data product above
self.assertTrue(dpd_ids is not None)
dp_ids, _ = self.rrclient.find_resources(restype=OT.DataProcess,
id_only=False)
# only one DP becuase the PFs that are in the code dataproduct above are not activated yet.
self.assertEquals(len(dp_ids), 1)
# validate the name and version label NEW SA - 2
dataprocessdef_obj = self.dataprocessclient.read_data_process_definition(
dataprocessdef_id)
self.assertEqual(dataprocessdef_obj.version_label, '1.0a')
self.assertEqual(dataprocessdef_obj.name, 'add_arrays')
# validate that the DPD has an attachment NEW SA - 21
attachment_ids, assoc_ids = self.rrclient.find_objects(
dataprocessdef_id, PRED.hasAttachment, RT.Attachment, True)
self.assertEqual(len(attachment_ids), 1)
attachment_obj = self.rrclient.read_attachment(attachment_ids[0])
log.debug('attachment: %s', attachment_obj)
# validate that the data process resource has input and output data products associated
# L4-CI-SA-RQ-364 and NEW SA-3
outproduct_ids, assoc_ids = self.rrclient.find_objects(
dataprocess_id, PRED.hasOutputProduct, RT.DataProduct, True)
self.assertEqual(len(outproduct_ids), 1)
inproduct_ids, assoc_ids = self.rrclient.find_objects(
dataprocess_id, PRED.hasInputProduct, RT.DataProduct, True)
self.assertEqual(len(inproduct_ids), 1)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 2)
self.assertTrue(self.input_dp_id in output_data_product_provenance[
output_data_product_id[0]]['parents'])
self.assertTrue(output_data_product_provenance[
output_data_product_id[0]]['parents'][self.input_dp_id]
['data_process_definition_id'] == dataprocessdef_id)
# NEW SA - 4 | Data processing shall include the appropriate data product algorithm name and version number in
# the metadata of each output data product created by the data product algorithm.
output_data_product_obj, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=False)
self.assertTrue(output_data_product_obj[0].name != None)
self.assertTrue(output_data_product_obj[0]._rev != None)
# retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=dataprocess_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_transform_worker subscription_obj: %s',
subscription_objs[0])
# create a queue to catch the published granules
self.subscription_id = self.pubsub_client.create_subscription(
name='parsed_subscription',
stream_ids=[self.stream_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_id)
stream_route = self.pubsub_client.read_stream_route(self.stream_id)
self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id,
stream_route=stream_route)
for n in range(1, 101):
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher.publish(rdt.to_granule())
# validate that the output granule is received and the updated value is correct
self.assertTrue(self.granule_verified.wait(self.wait_time))
# validate that the data process loaded into worker event is received (L4-CI-SA-RQ-182)
self.assertTrue(
self.worker_assigned_event_verified.wait(self.wait_time))
# validate that the data process create (with data product ids) event is received (NEW SA -42)
self.assertTrue(self.dp_created_event_verified.wait(self.wait_time))
# validate that the data process heartbeat event is received (for every hundred granules processed) (L4-CI-SA-RQ-182)
#this takes a while so set wait limit to large value
self.assertTrue(self.heartbeat_event_verified.wait(200))
# validate that the code from the transform function can be retrieve via inspect_data_process_definition
src = self.dataprocessclient.inspect_data_process_definition(
dataprocessdef_id)
self.assertIn('def add_arrays(a, b)', src)
# now delete the DPD and DP then verify that the resources are retired so that information required for provenance are still available
self.dataprocessclient.delete_data_process(dataprocess_id)
self.dataprocessclient.delete_data_process_definition(
dataprocessdef_id)
in_dp_objs, _ = self.rrclient.find_objects(
subject=dataprocess_id,
predicate=PRED.hasInputProduct,
object_type=RT.DataProduct,
id_only=True)
self.assertTrue(in_dp_objs is not None)
dpd_objs, _ = self.rrclient.find_subjects(
subject_type=RT.DataProcessDefinition,
predicate=PRED.hasDataProcess,
object=dataprocess_id,
id_only=True)
self.assertTrue(dpd_objs is not None)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_transform_worker_with_instrumentdevice(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# Create CTD Parsed as the initial data product
# create a stream definition for the data from the ctd simulator
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id)
# only ever need one device for testing purposes.
instDevice_obj, _ = self.rrclient.find_resources(
restype=RT.InstrumentDevice, name='test_ctd_device')
if instDevice_obj:
instDevice_id = instDevice_obj[0]._id
else:
instDevice_obj = IonObject(RT.InstrumentDevice,
name='test_ctd_device',
description="test_ctd_device",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=self.input_dp_id)
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process(
)
self.addCleanup(self.dataprocessclient.delete_data_process,
dataprocess_id)
self.addCleanup(self.dataprocessclient.delete_data_process_definition,
dataprocessdef_id)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 3)
self.assertTrue(self.input_dp_id in output_data_product_provenance[
output_data_product_id[0]]['parents'])
self.assertTrue(instDevice_id in output_data_product_provenance[
self.input_dp_id]['parents'])
self.assertTrue(output_data_product_provenance[instDevice_id]['type']
== 'InstrumentDevice')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_transform_worker_with_platformdevice(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# Create CTD Parsed as the initial data product
# create a stream definition for the data from the ctd simulator
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id)
# only ever need one device for testing purposes.
platform_device_obj, _ = self.rrclient.find_resources(
restype=RT.PlatformDevice, name='TestPlatform')
if platform_device_obj:
platform_device_id = platform_device_obj[0]._id
else:
platform_device_obj = IonObject(RT.PlatformDevice,
name='TestPlatform',
description="TestPlatform",
serial_number="12345")
platform_device_id = self.imsclient.create_platform_device(
platform_device=platform_device_obj)
self.damsclient.assign_data_product(
input_resource_id=platform_device_id,
data_product_id=self.input_dp_id)
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process(
)
self.addCleanup(self.dataprocessclient.delete_data_process,
dataprocess_id)
self.addCleanup(self.dataprocessclient.delete_data_process_definition,
dataprocessdef_id)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 3)
self.assertTrue(self.input_dp_id in output_data_product_provenance[
output_data_product_id[0]]['parents'])
self.assertTrue(platform_device_id in output_data_product_provenance[
self.input_dp_id]['parents'])
self.assertTrue(output_data_product_provenance[platform_device_id]
['type'] == 'PlatformDevice')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_event_transform_worker(self):
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# test that a data process (type: data-product-in / event-out) can be defined and launched.
# verify that event fields are correctly populated
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
# create the DPD and two DPs
self.event_data_process_id = self.create_event_data_processes()
# retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=self.event_data_process_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_event_transform_worker subscription_obj: %s',
subscription_objs[0])
# create a queue to catch the published granules
self.subscription_id = self.pubsub_client.create_subscription(
name='parsed_subscription',
stream_ids=[self.stream_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_id)
stream_route = self.pubsub_client.read_stream_route(self.stream_id)
self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id,
stream_route=stream_route)
self.start_event_transform_listener()
self.data_modified = Event()
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher.publish(rdt.to_granule())
self.assertTrue(self.event_verified.wait(self.wait_time))
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_bad_argument_map(self):
self._output_stream_ids = []
# test that a data process (type: data-product-in / data-product-out) parameter mapping it validated during
# data process creation and that the correct exception is raised for both input and output.
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# two data processes using one transform and one DPD
dp1_func_output_dp_id = self.create_output_data_product()
# Set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri=
'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id, add_array_dpd_id, binding='add_array_func')
# create the data process with invalid argument map
argument_map = {"arr1": "foo", "arr2": "bar"}
output_param = "salinity"
with self.assertRaises(BadRequest) as cm:
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id],
argument_map=argument_map,
out_param_name=output_param)
ex = cm.exception
log.debug(' exception raised: %s', cm)
self.assertEqual(
ex.message,
"Input data product does not contain the parameters defined in argument map"
)
# create the data process with invalid output parameter name
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "foo"
with self.assertRaises(BadRequest) as cm:
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id],
argument_map=argument_map,
out_param_name=output_param)
ex = cm.exception
log.debug(' exception raised: %s', cm)
self.assertEqual(
ex.message,
"Output data product does not contain the output parameter name provided"
)
def create_event_data_processes(self):
# two data processes using one transform and one DPD
argument_map = {"a": "salinity"}
# set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='validate_salinity_array',
description='validate_salinity_array',
function='validate_salinity_array',
module="ion.processes.data.transforms.test.test_transform_worker",
arguments=['a'],
function_type=TransformFunctionType.TRANSFORM)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='validate_salinity_array',
description='validate_salinity_array',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id,
add_array_dpd_id,
binding='validate_salinity_array')
# create the data process
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=None,
argument_map=argument_map)
self.damsclient.register_process(dp1_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process,
dp1_data_process_id)
return dp1_data_process_id
def create_data_process(self):
# two data processes using one transform and one DPD
dp1_func_output_dp_id = self.create_output_data_product()
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "salinity"
# set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri=
'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
version_label='1.0a')
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id, add_array_dpd_id, binding='add_array_func')
# create the data process
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id],
argument_map=argument_map,
out_param_name=output_param)
self.damsclient.register_process(dp1_data_process_id)
#self.addCleanup(self.dataprocessclient.delete_data_process, dp1_data_process_id)
# add an attachment object to this DPD to test new SA-21
import msgpack
attachment_content = 'foo bar'
attachment_obj = IonObject(RT.Attachment,
name='test_attachment',
attachment_type=AttachmentType.ASCII,
content_type='text/plain',
content=msgpack.packb(attachment_content))
att_id = self.rrclient.create_attachment(add_array_dpd_id,
attachment_obj)
self.addCleanup(self.rrclient.delete_attachment, att_id)
return add_array_dpd_id, dp1_data_process_id, dp1_func_output_dp_id
def create_output_data_product(self):
dp1_outgoing_stream_id = self.pubsub_client.create_stream_definition(
name='dp1_stream', parameter_dictionary_id=self.parameter_dict_id)
dp1_output_dp_obj = IonObject(RT.DataProduct,
name='data_process1_data_product',
description='output of add array func')
dp1_func_output_dp_id = self.dataproductclient.create_data_product(
dp1_output_dp_obj, dp1_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product,
dp1_func_output_dp_id)
# retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp1_func_output_dp_id,
PRED.hasStream, None, True)
self._output_stream_ids.append(stream_ids[0])
subscription_id = self.pubsub_client.create_subscription(
'validator', data_product_ids=[dp1_func_output_dp_id])
self.addCleanup(self.pubsub_client.delete_subscription,
subscription_id)
def on_granule(msg, route, stream_id):
log.debug('recv_packet stream_id: %s route: %s msg: %s',
stream_id, route, msg)
self.validate_output_granule(msg, route, stream_id)
self.granule_verified.set()
validator = StandaloneStreamSubscriber('validator',
callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
subscription_id)
return dp1_func_output_dp_id
def validate_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
data_process_event = args[0]
log.debug("DataProcessStatusEvent: %s",
str(data_process_event.__dict__))
# if data process already created, check origin
if self.dp_list:
self.assertIn(data_process_event.origin, self.dp_list)
# if this is a heartbeat event then 100 granules have been processed
if 'data process status update.' in data_process_event.description:
self.heartbeat_event_verified.set()
else:
# else check that this is the assign event
if 'Data process assigned to transform worker' in data_process_event.description:
self.worker_assigned_event_verified.set()
elif 'Data process created for data product' in data_process_event.description:
self.dp_created_event_verified.set()
def validate_output_granule(self, msg, route, stream_id):
self.assertIn(stream_id, self._output_stream_ids)
rdt = RecordDictionaryTool.load_from_granule(msg)
log.debug('validate_output_granule rdt: %s', rdt)
sal_val = rdt['salinity']
np.testing.assert_array_equal(sal_val, np.array([3]))
def start_event_listener(self):
es = EventSubscriber(event_type=OT.DataProcessStatusEvent,
callback=self.validate_event)
es.start()
self.addCleanup(es.stop)
def validate_transform_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
status_alert_event = args[0]
np.testing.assert_array_equal(status_alert_event.origin,
self.stream_id)
np.testing.assert_array_equal(status_alert_event.values,
np.array([self.event_data_process_id]))
log.debug("DeviceStatusAlertEvent: %s",
str(status_alert_event.__dict__))
self.event_verified.set()
def start_event_transform_listener(self):
es = EventSubscriber(event_type=OT.DeviceStatusAlertEvent,
callback=self.validate_transform_event)
es.start()
self.addCleanup(es.stop)
def test_download(self):
egg_url = 'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
egg_path = TransformWorker.download_egg(egg_url)
import pkg_resources
pkg_resources.working_set.add_entry(egg_path)
from ion_example.add_arrays import add_arrays
a = add_arrays(1, 2)
self.assertEquals(a, 3)
class TestIntDataAcquisitionManagementService(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataAcquisitionManagementService
self.client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
def tearDown(self):
pass
#@unittest.skip('Not done yet.')
def test_data_source_ops(self):
# test creating a new data source
print 'Creating new data source'
datasource_obj = IonObject(RT.DataSource,
name='DataSource1',
description='instrument based new source' ,
type='sbe37')
try:
ds_id = self.client.create_data_source(datasource_obj)
except BadRequest as ex:
self.fail("failed to create new data source: %s" %ex)
print 'new data source id = ', ds_id
# test reading a non-existent data source
print 'reading non-existent data source'
try:
dp_obj = self.client.read_data_source('some_fake_id')
except NotFound as ex:
pass
else:
self.fail("non-existing data source was found during read: %s" %dp_obj)
# update a data source (tests read also)
print 'Updating data source'
# first get the existing data source object
try:
datasource_obj = self.client.read_data_source(ds_id)
except NotFound as ex:
self.fail("existing data source was not found during read")
else:
pass
# now tweak the object
datasource_obj.description = 'the very first data source'
# now write the dp back to the registry
try:
update_result = self.client.update_data_source(datasource_obj)
except NotFound as ex:
self.fail("existing data source was not found during update")
except Conflict as ex:
self.fail("revision conflict exception during data source update")
#else:
# self.assertTrue(update_result == True)
# now get the data source back to see if it was updated
try:
datasource_obj = self.client.read_data_source(ds_id)
except NotFound as ex:
self.fail("existing data source was not found during read")
else:
pass
self.assertTrue(datasource_obj.description == 'the very first data source')
# now 'delete' the data source
print "deleting data source"
try:
delete_result = self.client.force_delete_data_source(ds_id)
except NotFound as ex:
self.fail("existing data source was not found during delete")
#self.assertTrue(delete_result == True)
# now try to get the deleted dp object
try:
dp_obj = self.client.read_data_source(ds_id)
except NotFound as ex:
pass
else:
self.fail("deleted data source was found during read")
# now try to delete the already deleted data source object
print "deleting non-existing data source"
try:
delete_result = self.client.delete_data_source(ds_id)
except NotFound as ex:
pass
else:
self.fail("non-existing data source was found during delete")
#@unittest.skip('Not done yet.')
def test_register_instrument(self):
# Register an instrument as a data producer in coordination with DM PubSub: create stream, register and create producer object
# set up initial instrument to register
instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product')
instrument_id, rev = self.rrclient.create(instrument_obj)
dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product')
dataproduct_id, rev = self.rrclient.create(dataproduct_obj)
# test registering a new data producer
try:
ds_id = self.client.register_instrument(instrument_id)
except BadRequest as ex:
self.fail("failed to create new data producer: %s" %ex)
print 'new data producer id = ', ds_id
# test assigning a data product to an instrument, creating the stream for the product
try:
self.client.assign_data_product(instrument_id, dataproduct_id)
except BadRequest as ex:
self.fail("failed to assign data product to data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to assign data product to data producer: %s" %ex)
# test UNassigning a data product from instrument, deleting the stream for the product
try:
self.client.unassign_data_product(instrument_id, dataproduct_id)
except BadRequest as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
# test UNregistering a new data producer
try:
ds_id = self.client.unregister_instrument(instrument_id)
except NotFound as ex:
self.fail("failed to unregister instrument producer: %s" %ex)
def test_register_external_data_set(self):
# Register an external data set as a data producer in coordination with DM PubSub: create stream, register and create producer object
# set up initial instrument to register
ext_dataset_obj = IonObject(RT.ExternalDataset, name='DataSet1',description='an external data feed')
ext_dataset_id, rev = self.rrclient.create(ext_dataset_obj)
dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product')
dataproduct_id, rev = self.rrclient.create(dataproduct_obj)
# test registering a new external data set
try:
ds_id = self.client.register_external_data_set(ext_dataset_id)
except BadRequest as ex:
self.fail("failed to create new data producer: %s" %ex)
print 'new data producer id = ', ds_id
# test assigning a data product to an ext_dataset_id, creating the stream for the product
try:
self.client.assign_data_product(ext_dataset_id, dataproduct_id)
except BadRequest as ex:
self.fail("failed to assign data product to data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to assign data product to data producer: %s" %ex)
# test UNassigning a data product from ext_dataset_id, deleting the stream for the product
try:
self.client.unassign_data_product(ext_dataset_id, dataproduct_id)
except BadRequest as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
# test UNregistering a external data set
try:
ds_id = self.client.unregister_external_data_set(ext_dataset_id)
except NotFound as ex:
self.fail("failed to unregister instrument producer: %s" %ex)
#@unittest.skip('not ready')
def test_eoi_resources(self):
#
# test creating a new data provider
#
print 'Creating new external_data_provider'
dataprovider_obj = IonObject(RT.ExternalDataProvider,
name='ExtDataProvider1',
description='external data provider ')
try:
dataprovider_id = self.client.create_external_data_provider(dataprovider_obj)
except BadRequest as ex:
self.fail("failed to create new data provider: %s" %ex)
print 'new data provider id = ', dataprovider_id
#
# test creating a new data source
#
print 'Creating new data source'
datasource_obj = IonObject(RT.DataSource,
name='DataSource1',
description='data source ',
type='DAP')
try:
datasource_id = self.client.create_data_source(datasource_obj)
except BadRequest as ex:
self.fail("failed to create new data source: %s" %ex)
print 'new data source id = ', datasource_id
#
# test creating a new data source model
#
print 'Creating new data source model'
datamodel_obj = IonObject(RT.DataSourceModel,
name='DataSourceModel1',
description='data source model')
try:
datamodel_id = self.client.create_data_source_model(datamodel_obj)
except BadRequest as ex:
self.fail("failed to create new data source model: %s" %ex)
print 'new data source model id = ', datamodel_id
#
# test creating a new external data set
#
print 'Creating new external data set'
dataset_obj = IonObject(RT.ExternalDataset,
name='ExternalDataSet1',
description='external data set ')
try:
extdataset_id = self.client.create_external_dataset(dataset_obj)
except BadRequest as ex:
self.fail("failed to create new external data set: %s" %ex)
print 'new external data set id = ', extdataset_id
#
# test creating a new dataset agent instance
#
print 'Creating new external data agent '
datasetagent_obj = IonObject(RT.ExternalDatasetAgent,
name='ExternalDatasetAgent1',
description='external data agent ',
handler_module = 'module_name',
handler_class = 'class_name')
try:
datasetagent_id = self.client.create_external_dataset_agent(datasetagent_obj)
except BadRequest as ex:
self.fail("failed to create new external dataset agent: %s" %ex)
print 'new external data agent id = ', datasetagent_id
#
# test creating a new datasource agent instance
#
print 'Creating new data source agent '
datasourceagent_obj = IonObject(RT.DataSourceAgent,
name='DataSourceAgent1',
description=' DataSource agent ')
try:
datasource_agent_id = self.client.create_data_source_agent(datasourceagent_obj)
except BadRequest as ex:
self.fail("failed to create new external datasource agent: %s" %ex)
print 'new external data agent id = ', datasource_agent_id
#
# test creating a new dataset agent instance
#
print 'Creating new external dataset agent instance'
datasetagentinstance_obj = IonObject(RT.ExternalDatasetAgentInstance,
name='ExternalDatasetAgentInstance1',
description='external dataset agent instance ')
try:
datasetagentinstance_id = self.client.create_external_dataset_agent_instance(datasetagentinstance_obj, datasetagent_id)
except BadRequest as ex:
self.fail("failed to create new external dataset agent instance: %s" %ex)
print 'new external data agent instance id = ', datasetagentinstance_id
#
# test creating a new datasource agent instance
#
print 'Creating new data source agent '
datasourceagentinstance_obj = IonObject(RT.DataSourceAgentInstance,
name='ExternalDataSourceAgentInstance1',
description='external DataSource agent instance ')
try:
datasource_agent_instance_id = self.client.create_data_source_agent_instance(datasourceagentinstance_obj)
except BadRequest as ex:
self.fail("failed to create new external datasource agent instance: %s" %ex)
print 'new external data agent id = ', datasource_agent_instance_id
#
# test assign / unassign
#
self.client.unassign_data_source_from_external_data_provider(datasource_id, dataprovider_id)
self.client.unassign_data_source_from_data_model(datasource_id, datamodel_id)
self.client.unassign_external_dataset_from_data_source(extdataset_id, datasource_id)
#
# test read
#
try:
dp_obj = self.client.read_external_data_provider(dataprovider_id)
except NotFound as ex:
self.fail("existing data provicer was not found during read")
else:
pass
try:
dp_obj = self.client.read_data_source(datasource_id)
except NotFound as ex:
self.fail("existing data source was not found during read")
else:
pass
#
# test delete
#
try:
self.client.delete_external_data_provider(dataprovider_id)
self.client.delete_data_source(datasource_id)
self.client.delete_external_dataset(extdataset_id)
self.client.delete_data_source_model(datamodel_id)
self.client.delete_external_dataset_agent(datasetagent_id)
self.client.delete_data_source_agent(datasource_agent_instance_id)
self.client.force_delete_external_data_provider(dataprovider_id)
self.client.force_delete_data_source(datasource_id)
self.client.force_delete_external_dataset(extdataset_id)
self.client.force_delete_data_source_model(datamodel_id)
self.client.force_delete_external_dataset_agent(datasetagent_id)
self.client.force_delete_data_source_agent(datasource_agent_instance_id)
except NotFound as ex:
self.fail("existing data product was not found during delete")
# test reading a non-existent data product
print 'reading non-existent data product'
try:
bad_obj = self.client.read_external_data_provider('some_fake_id')
except NotFound as ex:
pass
else:
self.fail("non-existing data product was found during read: %s" %bad_obj)
class IndexManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2dm.yml")
self.ims_cli = IndexManagementServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
self.index_name = "test_index"
def test_create_elasticsearch_index(self):
index_name = self.index_name
ims_cli = self.ims_cli
rr_cli = self.rr_cli
options = SearchOptions()
options.attribute_match = ["test_field"]
index_id = ims_cli.create_index(
name=index_name, content_type=IndexManagementService.ELASTICSEARCH_INDEX, options=options
)
index_result = self.rr_cli.read(index_id)
self.assertIsInstance(index_result, ElasticSearchIndex)
self.assertTrue(index_result.name == index_name)
# ======================================
# Clean up
# ======================================
rr_cli.delete(index_id)
def test_create_couchdb_index(self):
index_name = self.index_name
ims_cli = self.ims_cli
rr_cli = self.rr_cli
options = SearchOptions()
options.attribute_match = ["name"]
index_id = ims_cli.create_index(
index_name,
content_type=IndexManagementService.COUCHDB_INDEX,
options=options,
datastore_name="fake",
view_name="fake/by_fake",
)
index_result = self.rr_cli.read(index_id)
self.assertIsInstance(index_result, CouchDBIndex)
self.assertTrue(index_result.name == index_name)
# ======================================
# Clean up
# ======================================
rr_cli.delete(index_id)
def test_read_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
index = ims_cli.read_index(index_id)
self.assertIsInstance(index, Index)
self.assertTrue(index.name == index_name)
rr_cli.delete(index_id)
def test_delete_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
ims_cli.delete_index(index_id)
with self.assertRaises(NotFound):
rr_cli.delete(index_id)
def test_update_index(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
index_res = Index(name=index_name)
index_id, _ = rr_cli.create(index_res)
index = ims_cli.read_index(index_id)
index.name = "another"
ims_cli.update_index(index)
index = rr_cli.read(index_id)
self.assertTrue(index.name == "another")
def test_find_indexes(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
# ======================================
# Index Pool
# ======================================
indexes = [Index(name="first"), Index(name="second"), Index(name="third")]
id_pool = list()
for index in indexes:
id_pool.append(rr_cli.create(index)[0])
index_id = ims_cli.find_indexes(index_name="second")
index = ims_cli.read_index(index_id)
self.assertTrue(index.name == "second")
# ======================================
# Clean up
# ======================================
for index_id in id_pool:
rr_cli.delete(index_id)
def test_list_indexes(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
index_name = self.index_name
# ======================================
# Index Pool
# ======================================
indexes = [Index(name="first"), Index(name="second"), Index(name="third")]
id_pool = list()
for index in indexes:
id_pool.append(rr_cli.create(index)[0])
names = set(ims_cli.list_indexes().keys())
self.assertTrue(names == set(["first", "second", "third"]))
# ======================================
# Clean up
# ======================================
for index_id in id_pool:
rr_cli.delete(index_id)
def test_create_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
with self.assertRaises(BadRequest):
ims_cli.create_collection("failing_collection")
resources = [Resource(), Resource(), Resource()]
resources = [rr_cli.create(i)[0] for i in resources]
collection_id = ims_cli.create_collection("working_collection", resources)
collection = rr_cli.read(collection_id)
collection_resources = ims_cli.list_collection_resources(collection_id, id_only=True)
self.assertTrue(
set(collection_resources) == set(resources), "%s != %s" % (set(collection_resources), set(resources))
)
def test_read_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
collection = Collection(name="working_collection")
collection_id, _ = rr_cli.create(collection)
collection = ims_cli.read_collection(collection_id)
self.assertTrue(collection.name == "working_collection")
def test_update_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
collection = Collection(name="useful_collection")
collection_id, _ = rr_cli.create(collection)
collection = rr_cli.read(collection_id)
collection.name = "nub"
ims_cli.update_collection(collection)
collection = rr_cli.read(collection_id)
self.assertTrue(collection.name == "nub")
def test_delete_collection(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
res = Resource()
res_id, rev = rr_cli.create(res)
collection_id = ims_cli.create_collection(name="test_collection", resources=[res_id])
ims_cli.delete_collection(collection_id)
with self.assertRaises(NotFound):
rr_cli.read(collection_id)
def test_list_collection_resources(self):
ims_cli = self.ims_cli
rr_cli = self.rr_cli
# ========================================
# Resource Pool
# ========================================
resources = [
InformationResource(name="bean_counter"),
InformationResource(name="lunar_rock"),
InformationResource("aperature"),
InformationResource("lemons"),
]
resources = [rr_cli.create(i)[0] for i in resources]
collection = Collection(name="park_bench")
collection_id = ims_cli.create_collection(name="park_bench", resources=resources)
retval = ims_cli.list_collection_resources(collection_id, id_only=True)
retval.sort()
resources.sort()
self.assertTrue(retval == resources, "%s != %s" % (retval, resources))
def test_find_collection(self):
res_id, _ = self.rr_cli.create(Resource(name="test_res"))
collection_id = self.ims_cli.create_collection("test", [res_id])
retval = self.ims_cli.find_collection(collection_name="test")
self.assertTrue(retval[0] == collection_id)
retval = self.ims_cli.find_collection(resource_ids=[res_id])
self.assertTrue(retval[0] == collection_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
print 'started services'
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
sensor_model_id #is only a target
#create a parsed product for this instrument output
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain = tdom,
spatial_domain = sdom)
pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id)
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name)
#check agent instance
inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id)
self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
#compound assoc return list of lists so check the first element
self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent[0].name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name)
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
#check data_product_parameters_set
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.data_product_parameters_set.status)
self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value)
# the ctd parameters should include 'temp'
self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical'])
#none of these will work because there is no agent
self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
extended_instrument.computed.firmware_version.status)
self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
extended_instrument.computed.operational_state.status)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.power_status_roll_up.status)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.communications_status_roll_up.status)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.data_status_roll_up.status)
self.assertEqual(StatusType.STATUS_OK,
extended_instrument.computed.data_status_roll_up.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.location_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.recent_events.status)
# self.assertEqual([], extended_instrument.computed.recent_events.value)
# cleanup
c = DotDict()
c.resource_registry = self.RR
resource_impl = ResourceImpl(c)
resource_impl.pluck(instrument_agent_id)
resource_impl.pluck(instrument_model_id)
resource_impl.pluck(instrument_device_id)
resource_impl.pluck(platform_agent_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
def test_custom_attributes(self):
"""
Test assignment of custom attributes
"""
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel,
{"custom_attributes":
{"favorite_color": "attr desc goes here"}
}))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice,
{"custom_attributes":
{"favorite_color": "red",
"bogus_attr": "should raise warning"
}
}))
self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
# cleanup
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
def _get_datastore(self, dataset_id):
dataset = self.DSC.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_checkpoint_restore(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel",
stream_configuration= {'raw': 'ctd_raw_param_dict' , 'parsed': 'ctd_parsed_param_dict' })
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver" )
instAgent_id = self.IMS.create_instrument_agent(instAgent_obj)
log.debug( 'new InstrumentAgent id = %s', instAgent_id)
self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 '
+ '(SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': 'sbe37-simulator.oceanobservatories.org',
'device_port': 4001,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'command_port': 4002,
'data_port': 4003,
'log_level': 5,
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
comms_device_address='sbe37-simulator.oceanobservatories.org',
comms_device_port=4001,
port_agent_config = port_agent_config)
instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary=spdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict')
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary=rpdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id,
parameter_dictionary=spdict_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
self._get_datastore(self.parsed_dataset)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id,
parameter_dictionary=rpdict_id)
log.debug( 'new dp_id = %s', str(data_product_id2))
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
# spin up agent
self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.IMS.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
instance_obj.agent_process_id)
# take snapshot of config
snap_id = self.IMS.agent_state_checkpoint(instDevice_id, "xyzzy snapshot")
snap_obj = self.RR.read_attachment(snap_id, include_content=True)
print "Saved config:"
print snap_obj.content
#modify config
instance_obj.driver_config["comms_config"] = "BAD_DATA"
self.RR.update(instance_obj)
#restore config
self.IMS.agent_state_restore(instDevice_id, snap_id)
instance_obj = self.RR.read(instAgentInstance_id)
self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"])
class TestIntDataAcquisitionManagementService(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataAcquisitionManagementService
self.client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
def tearDown(self):
pass
#@unittest.skip('Not done yet.')
def test_data_source_ops(self):
# test creating a new data source
print 'Creating new data source'
datasource_obj = IonObject(RT.DataSource,
name='DataSource1',
description='instrument based new source' ,
type='sbe37')
try:
ds_id = self.client.create_data_source(datasource_obj)
except BadRequest as ex:
self.fail("failed to create new data source: %s" %ex)
print 'new data source id = ', ds_id
# test reading a non-existent data source
print 'reading non-existent data source'
try:
dp_obj = self.client.read_data_source('some_fake_id')
except NotFound as ex:
pass
else:
self.fail("non-existing data source was found during read: %s" %dp_obj)
# update a data source (tests read also)
print 'Updating data source'
# first get the existing data source object
try:
datasource_obj = self.client.read_data_source(ds_id)
except NotFound as ex:
self.fail("existing data source was not found during read")
else:
pass
# now tweak the object
datasource_obj.description = 'the very first data source'
# now write the dp back to the registry
try:
update_result = self.client.update_data_source(datasource_obj)
except NotFound as ex:
self.fail("existing data source was not found during update")
except Conflict as ex:
self.fail("revision conflict exception during data source update")
#else:
# self.assertTrue(update_result == True)
# now get the data source back to see if it was updated
try:
datasource_obj = self.client.read_data_source(ds_id)
except NotFound as ex:
self.fail("existing data source was not found during read")
else:
pass
self.assertTrue(datasource_obj.description == 'the very first data source')
# now 'delete' the data source
print "deleting data source"
try:
delete_result = self.client.delete_data_source(ds_id)
except NotFound as ex:
self.fail("existing data source was not found during delete")
#self.assertTrue(delete_result == True)
# now try to get the deleted dp object
try:
dp_obj = self.client.read_data_source(ds_id)
except NotFound as ex:
pass
else:
self.fail("deleted data source was found during read")
# now try to delete the already deleted data source object
print "deleting non-existing data source"
try:
delete_result = self.client.delete_data_source(ds_id)
except NotFound as ex:
pass
else:
self.fail("non-existing data source was found during delete")
#@unittest.skip('Not done yet.')
def test_register_instrument(self):
# Register an instrument as a data producer in coordination with DM PubSub: create stream, register and create producer object
# set up initial instrument to register
instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product')
instrument_id, rev = self.rrclient.create(instrument_obj)
dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product')
dataproduct_id, rev = self.rrclient.create(dataproduct_obj)
# test registering a new data producer
try:
ds_id = self.client.register_instrument(instrument_id)
except BadRequest as ex:
self.fail("failed to create new data producer: %s" %ex)
print 'new data producer id = ', ds_id
# test assigning a data product to an instrument, creating the stream for the product
try:
self.client.assign_data_product(instrument_id, dataproduct_id, True)
except BadRequest as ex:
self.fail("failed to assign data product to data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to assign data product to data producer: %s" %ex)
# test UNassigning a data product from instrument, deleting the stream for the product
try:
self.client.unassign_data_product(instrument_id, dataproduct_id, True)
except BadRequest as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
# test UNregistering a new data producer
try:
ds_id = self.client.unregister_instrument(instrument_id)
except NotFound as ex:
self.fail("failed to unregister instrument producer: %s" %ex)
def test_register_external_data_set(self):
# Register an external data set as a data producer in coordination with DM PubSub: create stream, register and create producer object
# set up initial instrument to register
ext_dataset_obj = IonObject(RT.ExternalDataset, name='DataSet1',description='an external data feed')
ext_dataset_id, rev = self.rrclient.create(ext_dataset_obj)
dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product')
dataproduct_id, rev = self.rrclient.create(dataproduct_obj)
# test registering a new external data set
try:
ds_id = self.client.register_external_data_set(ext_dataset_id)
except BadRequest as ex:
self.fail("failed to create new data producer: %s" %ex)
print 'new data producer id = ', ds_id
# test assigning a data product to an ext_dataset_id, creating the stream for the product
try:
self.client.assign_data_product(ext_dataset_id, dataproduct_id, True)
except BadRequest as ex:
self.fail("failed to assign data product to data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to assign data product to data producer: %s" %ex)
# test UNassigning a data product from ext_dataset_id, deleting the stream for the product
try:
self.client.unassign_data_product(ext_dataset_id, dataproduct_id, True)
except BadRequest as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
# test UNregistering a external data set
try:
ds_id = self.client.unregister_external_data_set(ext_dataset_id)
except NotFound as ex:
self.fail("failed to unregister instrument producer: %s" %ex)
def test_register_process(self):
# Register a data process as a data producer in coordination with DM PubSub: create stream, register and create producer object
# set up initial instrument to register
process_obj = IonObject(RT.DataProcess, name='Proc1',description='a data process transform')
process_id, rev = self.rrclient.create(process_obj)
dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product')
dataproduct_id, rev = self.rrclient.create(dataproduct_obj)
# test registering a new process
try:
ds_id = self.client.register_process(process_id)
except BadRequest as ex:
self.fail("failed to create new data producer: %s" %ex)
print 'new data producer id = ', ds_id
# test assigning a data product to a process, no stream create
try:
self.client.assign_data_product(process_id, dataproduct_id, False)
except BadRequest as ex:
self.fail("failed to create new data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to create new data producer: %s" %ex)
# test UNassigning a data product from the data process, deleting the stream for the product
try:
self.client.unassign_data_product(process_id, dataproduct_id, False)
except BadRequest as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
except NotFound as ex:
self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex)
# test UNregistering a process
try:
ds_id = self.client.unregister_process(process_id)
except NotFound as ex:
self.fail("failed to unregister instrument producer: %s" %ex)
#@unittest.skip('not ready')
def test_eoi_resources(self):
#
# test creating a new data provider
#
print 'Creating new external_data_provider'
dataprovider_obj = IonObject(RT.ExternalDataProvider,
name='ExtDataProvider1',
description='external data provider ')
try:
dataprovider_id = self.client.create_external_data_provider(dataprovider_obj)
except BadRequest as ex:
self.fail("failed to create new data provider: %s" %ex)
print 'new data provider id = ', dataprovider_id
#
# test creating a new data source
#
print 'Creating new data source'
datasource_obj = IonObject(RT.DataSource,
name='DataSource1',
description='data source ',
type='DAP')
try:
datasource_id = self.client.create_data_source(datasource_obj)
except BadRequest as ex:
self.fail("failed to create new data source: %s" %ex)
print 'new data source id = ', datasource_id
#
# test creating a new data source model
#
print 'Creating new data source model'
datamodel_obj = IonObject(RT.DataSourceModel,
name='DataSourceModel1',
description='data source model',
model='model1')
try:
datamodel_id = self.client.create_data_source_model(datamodel_obj)
except BadRequest as ex:
self.fail("failed to create new data source model: %s" %ex)
print 'new data source model id = ', datamodel_id
#
# test creating a new external data set
#
print 'Creating new external data set'
dataset_obj = IonObject(RT.ExternalDataset,
name='ExternalDataSet1',
description='external data set ')
try:
extdataset_id = self.client.create_external_dataset(dataset_obj)
except BadRequest as ex:
self.fail("failed to create new external data set: %s" %ex)
print 'new external data set id = ', extdataset_id
#
# test creating a new dataset agent instance
#
print 'Creating new external data agent '
datasetagent_obj = IonObject(RT.ExternalDatasetAgent,
name='ExternalDatasetAgent1',
description='external data agent ')
try:
datasetagent_id = self.client.create_external_dataset_agent(datasetagent_obj)
except BadRequest as ex:
self.fail("failed to create new external dataset agent: %s" %ex)
print 'new external data agent id = ', datasetagent_id
#
# test creating a new datasource agent instance
#
print 'Creating new data source agent '
datasourceagent_obj = IonObject(RT.DataSourceAgent,
name='DataSourceAgent1',
description=' DataSource agent ')
try:
datasource_agent_id = self.client.create_data_source_agent(datasourceagent_obj)
except BadRequest as ex:
self.fail("failed to create new external datasource agent: %s" %ex)
print 'new external data agent id = ', datasource_agent_id
#
# test creating a new dataset agent instance
#
print 'Creating new external dataset agent instance'
datasetagentinstance_obj = IonObject(RT.ExternalDatasetAgentInstance,
name='ExternalDatasetAgentInstance1',
description='external dataset agent instance ')
try:
datasetagentinstance_id = self.client.create_external_dataset_agent_instance(datasetagentinstance_obj, datasetagent_id)
except BadRequest as ex:
self.fail("failed to create new external dataset agent instance: %s" %ex)
print 'new external data agent instance id = ', datasetagentinstance_id
#
# test creating a new datasource agent instance
#
print 'Creating new data source agent '
datasourceagentinstance_obj = IonObject(RT.DataSourceAgentInstance,
name='ExternalDataSourceAgentInstance1',
description='external DataSource agent instance ')
try:
datasource_agent_instance_id = self.client.create_data_source_agent_instance(datasourceagentinstance_obj)
except BadRequest as ex:
self.fail("failed to create new external datasource agent instance: %s" %ex)
print 'new external data agent id = ', datasource_agent_instance_id
#
# test assign / unassign
#
self.client.unassign_data_source_from_external_data_provider(datasource_id, dataprovider_id)
self.client.unassign_data_source_from_data_model(datasource_id, datamodel_id)
self.client.unassign_external_dataset_from_data_source(extdataset_id, datasource_id)
#
# test read
#
try:
dp_obj = self.client.read_external_data_provider(dataprovider_id)
except NotFound as ex:
self.fail("existing data provicer was not found during read")
else:
pass
try:
dp_obj = self.client.read_data_source(datasource_id)
except NotFound as ex:
self.fail("existing data source was not found during read")
else:
pass
#
# test delete
#
try:
self.client.delete_external_data_provider(dataprovider_id)
self.client.delete_data_source(datasource_id)
self.client.delete_external_dataset(extdataset_id)
except NotFound as ex:
self.fail("existing data product was not found during delete")
# test reading a non-existent data product
print 'reading non-existent data product'
try:
bad_obj = self.client.read_external_data_provider('some_fake_id')
except NotFound as ex:
pass
else:
self.fail("non-existing data product was found during read: %s" %bad_obj)
class UserNotificationIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.unsc = UserNotificationServiceClient(node=self.container.node)
self.rrc = ResourceRegistryServiceClient(node=self.container.node)
self.imc = IdentityManagementServiceClient(node=self.container.node)
def test_find_event_types_for_resource(self):
# create a dataset object in the RR to pass into the UNS method
dataset_object = IonObject(RT.DataSet, name="dataset1")
dataset_id, version = self.rrc.create(dataset_object)
# get the list of event types for the dataset
events = self.unsc.find_event_types_for_resource(dataset_id)
log.debug("dataset events = " + str(events))
if not events == ['dataset_supplement_added', 'dataset_change']:
self.fail("failed to return correct list of event types")
# try to pass in an id of a resource that doesn't exist (should fail)
try:
events = self.unsc.find_event_types_for_resource("bogus_id")
self.fail("failed to detect non-existant resource")
except:
pass
def test_create_two_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
notification_id1 = self.unsc.create_notification(notification_object1, user_id)
# create second notification
notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
notification_id2 = self.unsc.create_notification(notification_object2, user_id)
# read the notifications back and check that they are correct
n1 = self.unsc.read_notification(notification_id1)
if n1.name != notification_object1.name or \
n1.origin_list != notification_object1.origin_list or \
n1.events_list != notification_object1.events_list:
self.fail("notification was not correct")
n2 = self.unsc.read_notification(notification_id2)
if n2.name != notification_object2.name or \
n2.origin_list != notification_object2.origin_list or \
n2.events_list != notification_object2.events_list:
self.fail("notification was not correct")
def test_delete_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
notification1_id = self.unsc.create_notification(notification_object1, user_id)
# create second notification
notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
notification2_id = self.unsc.create_notification(notification_object2, user_id)
# delete both notifications
self.unsc.delete_notification(notification1_id)
self.unsc.delete_notification(notification2_id)
# check that the notifications are not there
try:
n1 = self.unsc.read_notification(notification1_id)
except:
try:
n2 = self.unsc.read_notification(notification2_id)
except:
return
self.fail("failed to delete notifications")
def test_find_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
self.unsc.create_notification(notification_object, user_id)
# try to find all notifications for user
notifications = self.unsc.find_notifications_by_user(user_id)
if len(notifications) != 2:
self.fail("failed to find all notifications")
def test_update_user_notification(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create a notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
notification_id = self.unsc.create_notification(notification_object, user_id)
# read back the notification and change it
notification = self.unsc.read_notification(notification_id)
notification.origin_list = ['Some_Resource_Agent_ID5']
self.unsc.update_notification(notification)
# read back the notification and check that it got changed
notification = self.unsc.read_notification(notification_id)
if notification.origin_list != ['Some_Resource_Agent_ID5']:
self.fail("failed to change notification")
def test_send_notification_emails(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
self.unsc.create_notification(notification_object, user_id)
# publish an event for each notification to generate the emails
# this can't be easily check in SW so need to check for these at the [email protected] account
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event")
de_publisher = EventPublisher("DataEvent")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event")
gevent.sleep(1)
def test_find_events(self):
# publish some events for the event repository
rle_publisher = EventPublisher("ResourceLifecycleEvent")
de_publisher = EventPublisher("DataEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event1")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event2")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event3")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event1")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event2")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event3")
# find all events for the originator 'Some_Resource_Agent_ID1'
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1')
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID1':
self.fail("failed to find correct events")
# find all events for the originator 'DataEvent'
events = self.unsc.find_events(type='DataEvent')
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'DataEvent':
self.fail("failed to find correct events")
# find 2 events for the originator 'Some_Resource_Agent_ID1'
events = self.unsc.find_events(origin='Some_Resource_Agent_ID2', limit=2)
if len(events) != 2:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID2':
self.fail("failed to find correct events")
# find all events for the originator 'Some_Resource_Agent_ID1' in reverse time order
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1', descending=True)
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID1':
self.fail("failed to find correct events")
class TestResourceRegistry(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url("res/deploy/r2coi.yml")
# Now create client to bank service
self.resource_registry_service = ResourceRegistryServiceClient(node=self.container.node)
def test_crud(self):
# Some quick registry tests
# Can't call new with fields that aren't defined in the object's schema
with self.assertRaises(TypeError) as cm:
IonObject("UserInfo", name="name", foo="bar")
self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'")
# Can't call new with fields that aren't defined in the object's schema
with self.assertRaises(TypeError) as cm:
IonObject("UserInfo", {"name": "name", "foo": "bar"})
self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'")
# Can't call new with fields that aren't defined in the object's schema
with self.assertRaises(TypeError) as cm:
IonObject("UserInfo", {"name": "name"}, foo="bar")
self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'")
# Instantiate an object
obj = IonObject("UserInfo", name="name")
# Can set attributes that aren't in the object's schema
with self.assertRaises(AttributeError) as cm:
setattr(obj, "foo", "bar")
self.assertTrue(cm.exception.message == "'UserInfo' object has no attribute 'foo'")
# Cam't call update with object that hasn't been persisted
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.update(obj)
self.assertTrue(cm.exception.message.startswith("Object does not have required '_id' or '_rev' attribute"))
# Persist object and read it back
obj_id, obj_rev = self.resource_registry_service.create(obj)
read_obj = self.resource_registry_service.read(obj_id)
# Cannot create object with _id and _rev fields pre-set
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create(read_obj)
self.assertTrue(cm.exception.message.startswith("Doc must not have '_id'"))
# Update object
read_obj.name = "John Doe"
self.resource_registry_service.update(read_obj)
# Update should fail with revision mismatch
with self.assertRaises(Conflict) as cm:
self.resource_registry_service.update(read_obj)
self.assertTrue(cm.exception.message.startswith("Object not based on most current version"))
# Re-read and update object
read_obj = self.resource_registry_service.read(obj_id)
self.resource_registry_service.update(read_obj)
# Delete object
self.resource_registry_service.delete(obj_id)
# Make sure read, update and delete report error
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.read(obj_id)
self.assertTrue(cm.exception.message.startswith("Object with id"))
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.update(read_obj)
self.assertTrue(cm.exception.message.startswith("Object with id"))
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.delete(obj_id)
self.assertTrue(cm.exception.message.startswith("Object with id"))
# Owner creation tests
user = IonObject("ActorIdentity", name="user")
uid, _ = self.resource_registry_service.create(user)
inst = IonObject("InstrumentDevice", name="instrument")
iid, _ = self.resource_registry_service.create(inst, headers={"ion-actor-id": str(uid)})
ids, _ = self.resource_registry_service.find_objects(iid, PRED.hasOwner, RT.ActorIdentity, id_only=True)
self.assertEquals(len(ids), 1)
assoc = self.resource_registry_service.read(ids[0])
self.resource_registry_service.delete(iid)
with self.assertRaises(NotFound) as ex:
assoc = self.resource_registry_service.read(iid)
def test_lifecycle(self):
att = IonObject("InstrumentDevice", name="mine", description="desc")
rid, rev = self.resource_registry_service.create(att)
att1 = self.resource_registry_service.read(rid)
self.assertEquals(att1.name, att.name)
self.assertEquals(att1.lcstate, LCS.DRAFT_PRIVATE)
new_state = self.resource_registry_service.execute_lifecycle_transition(rid, LCE.PLAN)
self.assertEquals(new_state, LCS.PLANNED_PRIVATE)
att2 = self.resource_registry_service.read(rid)
self.assertEquals(att2.lcstate, LCS.PLANNED_PRIVATE)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.execute_lifecycle_transition(rid, LCE.UNANNOUNCE)
self.assertTrue(
"type=InstrumentDevice, lcstate=PLANNED_PRIVATE has no transition for event unannounce"
in cm.exception.message
)
new_state = self.resource_registry_service.execute_lifecycle_transition(rid, LCE.DEVELOP)
self.assertEquals(new_state, LCS.DEVELOPED_PRIVATE)
self.assertRaises(
iex.BadRequest,
self.resource_registry_service.execute_lifecycle_transition,
resource_id=rid,
transition_event="NONE##",
)
self.resource_registry_service.set_lifecycle_state(rid, LCS.INTEGRATED_PRIVATE)
att1 = self.resource_registry_service.read(rid)
self.assertEquals(att1.lcstate, LCS.INTEGRATED_PRIVATE)
def test_association(self):
# Instantiate ActorIdentity object
actor_identity_obj = IonObject("ActorIdentity", name="name")
actor_identity_obj_id, actor_identity_obj_rev = self.resource_registry_service.create(actor_identity_obj)
read_actor_identity_obj = self.resource_registry_service.read(actor_identity_obj_id)
# Instantiate UserInfo object
user_info_obj = IonObject("UserInfo", name="name")
user_info_obj_id, user_info_obj_rev = self.resource_registry_service.create(user_info_obj)
read_user_info_obj = self.resource_registry_service.read(user_info_obj_id)
# Test create failures
with self.assertRaises(AttributeError) as cm:
self.resource_registry_service.create_association(actor_identity_obj_id, PRED.bogus, user_info_obj_id)
self.assertTrue(cm.exception.message == "bogus")
# Predicate not provided
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(actor_identity_obj_id, None, user_info_obj_id)
self.assertTrue(cm.exception.message == "Association must have all elements set")
# Bad association type
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, "bogustype"
)
self.assertTrue(cm.exception.message == "Unsupported assoc_type: bogustype")
# Subject id or object not provided
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(None, PRED.hasInfo, user_info_obj_id)
self.assertTrue(cm.exception.message == "Association must have all elements set")
# Object id or object not provided
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, None)
self.assertTrue(cm.exception.message == "Association must have all elements set")
# Bad subject id
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.create_association("bogus", PRED.hasInfo, user_info_obj_id)
self.assertTrue(cm.exception.message == "Object with id bogus does not exist.")
# Bad object id
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, "bogus")
self.assertTrue(cm.exception.message == "Object with id bogus does not exist.")
# _id missing from subject
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(actor_identity_obj, PRED.hasInfo, user_info_obj_id)
self.assertTrue(cm.exception.message == "Subject id or rev not available")
# _id missing from object
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj)
self.assertTrue(cm.exception.message == "Object id or rev not available")
# Wrong subject type
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(user_info_obj_id, PRED.hasInfo, user_info_obj_id)
self.assertTrue(cm.exception.message == "Illegal subject type UserInfo for predicate hasInfo")
# Wrong object type
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, actor_identity_obj_id
)
self.assertTrue(cm.exception.message == "Illegal object type ActorIdentity for predicate hasInfo")
# Create two different association types between the same subject and predicate
assoc_id1, assoc_rev1 = self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id
)
# Read object, subject
res_obj1 = self.resource_registry_service.read_object(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo)
self.assertEquals(res_obj1._id, user_info_obj_id)
res_obj1 = self.resource_registry_service.read_object(
actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, id_only=True
)
self.assertEquals(res_obj1, user_info_obj_id)
res_obj2 = self.resource_registry_service.read_subject(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id)
self.assertEquals(res_obj2._id, actor_identity_obj_id)
res_obj2 = self.resource_registry_service.read_subject(
RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, id_only=True
)
self.assertEquals(res_obj2, actor_identity_obj_id)
# Create a similar association to a specific revision
# TODO: This is not a supported case so far
assoc_id2, assoc_rev2 = self.resource_registry_service.create_association(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, "H2R"
)
# Search for associations (good cases)
ret1 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo)
ret3 = self.resource_registry_service.find_associations(None, PRED.hasInfo)
self.assertTrue(len(ret1) == len(ret2) == len(ret3))
self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id)
ret1 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, None, False
)
ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, id_only=False)
ret3 = self.resource_registry_service.find_associations(predicate=PRED.hasInfo, id_only=False)
self.assertTrue(ret1 == ret2 == ret3)
# Search for associations (good cases)
ret1 = self.resource_registry_service.find_associations(
read_actor_identity_obj, PRED.hasInfo, read_user_info_obj
)
ret2 = self.resource_registry_service.find_associations(read_actor_identity_obj, PRED.hasInfo)
ret3 = self.resource_registry_service.find_associations(None, PRED.hasInfo)
self.assertTrue(len(ret1) == len(ret2) == len(ret3))
self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id)
ret1 = self.resource_registry_service.find_associations(
actor_identity_obj_id, PRED.hasInfo, read_user_info_obj, None, True
)
ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, id_only=True)
ret3 = self.resource_registry_service.find_associations(predicate=PRED.hasInfo, id_only=True)
self.assertTrue(ret1 == ret2 == ret3)
# Search for associations (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_associations(None, None, None)
self.assertTrue(cm.exception.message == "Illegal parameters")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_associations(actor_identity_obj_id, None, None)
self.assertTrue(cm.exception.message == "Illegal parameters")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_associations(None, None, user_info_obj_id)
self.assertTrue(cm.exception.message == "Illegal parameters")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_associations(actor_identity_obj, None, user_info_obj_id)
self.assertTrue(cm.exception.message == "Object id not available in subject")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_associations(actor_identity_obj_id, None, user_info_obj)
self.assertTrue(cm.exception.message == "Object id not available in object")
# Find subjects (good cases)
subj_ret1 = self.resource_registry_service.find_subjects(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True)
subj_ret2 = self.resource_registry_service.find_subjects(
RT.ActorIdentity, PRED.hasInfo, read_user_info_obj, True
)
self.assertTrue(len(subj_ret1) == len(subj_ret2))
self.assertTrue(subj_ret1[0] == subj_ret2[0])
self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id)
subj_ret3 = self.resource_registry_service.find_subjects(None, PRED.hasInfo, user_info_obj_id, True)
subj_ret4 = self.resource_registry_service.find_subjects(None, None, read_user_info_obj, True)
self.assertTrue(len(subj_ret3) == len(subj_ret4))
self.assertTrue(subj_ret3[0] == subj_ret4[0])
self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id)
subj_ret5 = self.resource_registry_service.find_subjects(None, PRED.hasInfo, user_info_obj_id, False)
subj_ret6 = self.resource_registry_service.find_subjects(None, None, read_user_info_obj, False)
self.assertTrue(len(subj_ret5) == len(subj_ret6))
self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id)
self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id)
# Find subjects (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_subjects(None, None, None)
self.assertTrue(cm.exception.message == "Must provide object")
with self.assertRaises(AttributeError) as cm:
self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.bogus, user_info_obj_id, True)
self.assertTrue(cm.exception.message == "bogus")
ret = self.resource_registry_service.find_subjects(RT.UserInfo, PRED.hasCredentials, user_info_obj_id, True)
self.assertTrue(len(ret[0]) == 0)
ret = self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.hasInfo, user_info_obj_id, True)
self.assertTrue(len(ret[0]) == 0)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.hasInfo, user_info_obj, True)
self.assertTrue(cm.exception.message == "Object id not available in object")
# Find objects (good cases)
subj_ret1 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, True)
subj_ret2 = self.resource_registry_service.find_objects(
read_actor_identity_obj, PRED.hasInfo, RT.UserInfo, True
)
self.assertTrue(len(subj_ret1) == len(subj_ret2))
self.assertTrue(subj_ret1[0] == subj_ret2[0])
self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id)
subj_ret3 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, None, True)
subj_ret4 = self.resource_registry_service.find_objects(actor_identity_obj_id, None, None, True)
self.assertTrue(len(subj_ret3) == len(subj_ret4))
self.assertTrue(subj_ret3[0] == subj_ret4[0])
self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id)
subj_ret5 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, None, False)
subj_ret6 = self.resource_registry_service.find_objects(read_actor_identity_obj, None, None, False)
self.assertTrue(len(subj_ret5) == len(subj_ret6))
self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id)
self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id)
# Find objects (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_objects(None, None, None)
self.assertTrue(cm.exception.message == "Must provide subject")
with self.assertRaises(AttributeError) as cm:
self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.bogus, RT.UserCredentials, True)
self.assertTrue(cm.exception.message == "bogus")
ret = self.resource_registry_service.find_objects(
actor_identity_obj_id, PRED.hasCredentials, RT.ActorIdentity, True
)
self.assertTrue(len(ret[0]) == 0)
ret = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, RT.UserCredentials, True)
self.assertTrue(len(ret[0]) == 0)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_objects(actor_identity_obj, PRED.hasInfo, RT.UserInfo, True)
self.assertTrue(cm.exception.message == "Object id not available in subject")
# Get association (bad cases)
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.get_association(None, None, None)
self.assertTrue(cm.exception.message == "Illegal parameters")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.get_association(actor_identity_obj_id, None, None)
self.assertTrue(cm.exception.message == "Illegal parameters")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.get_association(None, None, user_info_obj_id)
self.assertTrue(cm.exception.message == "Illegal parameters")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.get_association(actor_identity_obj, None, user_info_obj_id)
self.assertTrue(cm.exception.message == "Object id not available in subject")
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.get_association(actor_identity_obj_id, None, user_info_obj)
self.assertTrue(cm.exception.message == "Object id not available in object")
# Delete one of the associations
self.resource_registry_service.delete_association(assoc_id2)
assoc = self.resource_registry_service.get_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id)
self.assertTrue(assoc._id == assoc_id1)
# Delete (bad cases)
with self.assertRaises(NotFound) as cm:
self.resource_registry_service.delete_association("bogus")
self.assertTrue(cm.exception.message == "Object with id bogus does not exist.")
# Delete other association
self.resource_registry_service.delete_association(assoc_id1)
# Delete resources
self.resource_registry_service.delete(actor_identity_obj_id)
self.resource_registry_service.delete(user_info_obj_id)
def test_find_resources(self):
with self.assertRaises(BadRequest) as cm:
self.resource_registry_service.find_resources(RT.UserInfo, LCS.DRAFT, "name", False)
self.assertTrue(cm.exception.message == "find by name does not support lcstate")
ret = self.resource_registry_service.find_resources(RT.UserInfo, None, "name", False)
self.assertTrue(len(ret[0]) == 0)
# Instantiate an object
obj = IonObject("InstrumentDevice", name="name")
# Persist object and read it back
obj_id, obj_rev = self.resource_registry_service.create(obj)
read_obj = self.resource_registry_service.read(obj_id)
ret = self.resource_registry_service.find_resources(RT.InstrumentDevice, None, "name", False)
self.assertTrue(len(ret[0]) == 1)
self.assertTrue(ret[0][0]._id == read_obj._id)
ret = self.resource_registry_service.find_resources(RT.InstrumentDevice, LCS.DRAFT, None, False)
self.assertTrue(len(ret[0]) == 1)
self.assertTrue(ret[0][0]._id == read_obj._id)
# @attr('INT', group='coirr1')
# class TestResourceRegistry1(IonIntegrationTestCase):
#
# def setUp(self):
# # Start container
# self._start_container()
# self.container.start_rel_from_url('res/deploy/r2coi.yml')
#
# # Now create client to bank service
# self.resource_registry_service = ResourceRegistryServiceClient(node=self.container.node)
def test_attach(self):
binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82"
# Owner creation tests
instrument = IonObject("InstrumentDevice", name="instrument")
iid, _ = self.resource_registry_service.create(instrument)
att = Attachment(content=binary, attachment_type=AttachmentType.BLOB)
aid1 = self.resource_registry_service.create_attachment(iid, att)
att1 = self.resource_registry_service.read_attachment(aid1)
self.assertEquals(binary, att1.content)
import base64
att = Attachment(content=base64.encodestring(binary), attachment_type=AttachmentType.ASCII)
aid2 = self.resource_registry_service.create_attachment(iid, att)
att1 = self.resource_registry_service.read_attachment(aid2)
self.assertEquals(binary, base64.decodestring(att1.content))
att_ids = self.resource_registry_service.find_attachments(iid, id_only=True)
self.assertEquals(att_ids, [aid1, aid2])
att_ids = self.resource_registry_service.find_attachments(iid, id_only=True, descending=True)
self.assertEquals(att_ids, [aid2, aid1])
att_ids = self.resource_registry_service.find_attachments(iid, id_only=True, descending=True, limit=1)
self.assertEquals(att_ids, [aid2])
atts = self.resource_registry_service.find_attachments(iid, id_only=False, limit=1)
self.assertEquals(atts[0].content, att1.content)
self.resource_registry_service.delete_attachment(aid1)
att_ids = self.resource_registry_service.find_attachments(iid, id_only=True)
self.assertEquals(att_ids, [aid2])
self.resource_registry_service.delete_attachment(aid2)
att_ids = self.resource_registry_service.find_attachments(iid, id_only=True)
self.assertEquals(att_ids, [])
class UserNotificationIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.unsc = UserNotificationServiceClient(node=self.container.node)
self.rrc = ResourceRegistryServiceClient(node=self.container.node)
self.imc = IdentityManagementServiceClient(node=self.container.node)
def xtest_find_event_types_for_resource(self):
dataset_object = IonObject(RT.DataSet, name="dataset1")
dataset_id, version = self.rrc.create(dataset_object)
events = self.unsc.find_event_types_for_resource(dataset_id)
log.debug("dataset events = " + str(events))
try:
events = self.unsc.find_event_types_for_resource("bogus_id")
self.fail("failed to detect non-existant resource")
except:
pass
def test_create_two_user_notifications(self):
user_identty_object = IonObject(RT.UserIdentity, name="user1")
user_id = self.imc.create_user_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['resource_lifecycle']})
self.unsc.create_notification(notification_object, user_id)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['data']})
self.unsc.create_notification(notification_object, user_id)
def test_delete_user_notifications(self):
user_identty_object = IonObject(RT.UserIdentity, name="user1")
user_id = self.imc.create_user_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['resource_lifecycle']})
notification1_id = self.unsc.create_notification(notification_object1, user_id)
notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['data']})
notification2_id = self.unsc.create_notification(notification_object2, user_id)
self.unsc.delete_notification(notification1_id)
self.unsc.delete_notification(notification2_id)
def test_find_user_notifications(self):
user_identty_object = IonObject(RT.UserIdentity, name="user1")
user_id = self.imc.create_user_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['resource_lifecycle']})
self.unsc.create_notification(notification_object, user_id)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['data']})
self.unsc.create_notification(notification_object, user_id)
notifications = self.unsc.find_notifications_by_user(user_id)
for n in notifications:
log.debug("n = " +str(n))
def test_update_user_notification(self):
user_identty_object = IonObject(RT.UserIdentity, name="user1")
user_id = self.imc.create_user_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['resource_lifecycle']})
notification_id = self.unsc.create_notification(notification_object, user_id)
notification = self.rrc.read(notification_id)
notification.origin_list = ['Some_Resource_Agent_ID5']
self.unsc.update_notification(notification)
def test_send_notification_emails(self):
user_identty_object = IonObject(RT.UserIdentity, name="user1")
user_id = self.imc.create_user_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['resource_lifecycle']})
self.unsc.create_notification(notification_object, user_id)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['data']})
self.unsc.create_notification(notification_object, user_id)
rle_publisher = ResourceLifecycleEventPublisher()
rle_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event")
de_publisher = DataEventPublisher()
de_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID2', description="DE test event")
gevent.sleep(1)
def test_find_events(self):
rle_publisher = ResourceLifecycleEventPublisher(event_repo=self.container.event_repository)
rle_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event1")
rle_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event2")
de_publisher = DataEventPublisher(event_repo=self.container.event_repository)
de_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID2', description="DE test event1")
de_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID2', description="DE test event2")
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1')
for event in events:
log.debug("event=" + str(event))
events = self.unsc.find_events(type='DataEvent')
for event in events:
log.debug("event=" + str(event))
class TestDataProductProvenance(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def test_get_data_product_provenance_report(self):
#Create a test device
device_obj = Device(name='Device1',
description='test instrument site')
device_id, _ = self.rrclient.create(device_obj)
self.addCleanup(self.rrclient.delete, device_id)
#Create a test DataProduct
data_product1_obj = DataProduct(name='DataProduct1',
description='test data product 1')
data_product1_id, _ = self.rrclient.create(data_product1_obj)
self.addCleanup(self.rrclient.delete, data_product1_id)
#Create a test DataProcess
data_process_obj = DataProcess(name='DataProcess',
description='test data process')
data_process_id, _ = self.rrclient.create(data_process_obj)
self.addCleanup(self.rrclient.delete, data_process_id)
#Create a second test DataProduct
data_product2_obj = DataProduct(name='DataProduct2',
description='test data product 2')
data_product2_id, _ = self.rrclient.create(data_product2_obj)
self.addCleanup(self.rrclient.delete, data_product2_id)
#Create a test DataProducer
data_producer_obj = DataProducer(name='DataProducer',
description='test data producer')
data_producer_id, rev = self.rrclient.create(data_producer_obj)
#Link the DataProcess to the second DataProduct manually
assoc_id, _ = self.rrclient.create_association(subject=data_process_id, predicate=PRED.hasInputProduct, object=data_product2_id)
self.addCleanup(self.rrclient.delete_association, assoc_id)
# Register the instrument and process. This links the device and the data process
# with their own producers
self.damsclient.register_instrument(device_id)
self.addCleanup(self.damsclient.unregister_instrument, device_id)
self.damsclient.register_process(data_process_id)
self.addCleanup(self.damsclient.unregister_process, data_process_id)
#Manually link the first DataProduct with the test DataProducer
assoc_id, _ = self.rrclient.create_association(subject=data_product1_id, predicate=PRED.hasDataProducer, object=data_producer_id)
#Get the DataProducer linked to the DataProcess (created in register_process above)
#Associate that with with DataProduct1's DataProducer
data_process_producer_ids, _ = self.rrclient.find_objects(subject=data_process_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True)
assoc_id, _ = self.rrclient.create_association(subject=data_process_producer_ids[0], predicate=PRED.hasParent, object=data_producer_id)
self.addCleanup(self.rrclient.delete_association, assoc_id)
#Get the DataProducer linked to the Device (created in register_instrument
#Associate that with the DataProcess's DataProducer
device_producer_ids, _ = self.rrclient.find_objects(subject=device_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True)
assoc_id, _ = self.rrclient.create_association(subject=data_producer_id, predicate=PRED.hasParent, object=device_producer_ids[0])
#Create the links between the Device, DataProducts, DataProcess, and all DataProducers
self.damsclient.assign_data_product(input_resource_id=device_id, data_product_id=data_product1_id)
self.addCleanup(self.damsclient.unassign_data_product, device_id, data_product1_id)
self.damsclient.assign_data_product(input_resource_id=data_process_id, data_product_id=data_product2_id)
self.addCleanup(self.damsclient.unassign_data_product, data_process_id, data_product2_id)
#Traverse through the relationships to get the links between objects
res = self.dpmsclient.get_data_product_provenance_report(data_product2_id)
#Make sure there are four keys
self.assertEqual(len(res.keys()), 4)
parent_count = 0
config_count = 0
for v in res.itervalues():
if 'parent' in v:
parent_count += 1
if 'config' in v:
config_count += 1
#Make sure there are three parents and four configs
self.assertEqual(parent_count, 3)
self.assertEqual(config_count, 4)
@unittest.skip('This test is obsolete with new framework')
def test_get_provenance(self):
#create a deployment with metadata and an initial site and device
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "")
log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id))
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id))
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id)
# Create InstrumentAgent
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict' )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations = [parsed_config] )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
#-------------------------------
# Create CTD Parsed data product
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
#-------------------------------
# create a data product for the site to pass the OMS check.... we need to remove this check
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id)
#-------------------------------
# Deploy instrument device to instrument site
#-------------------------------
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id)
self.imsclient.deploy_instrument_device(instDevice_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
self.omsclient.activate_deployment(deployment_id)
inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False)
log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) )
#-------------------------------
# Create the agent instance
#-------------------------------
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
log.debug("TestDataProductProvenance: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' )
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' )
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id)
log.debug("TestDataProductProvenance: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' )
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' )
log.debug("TestDataProductProvenance: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id)
log.debug("TestDataProductProvenance: create output data product L2 Density")
# ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
# name='L2_Density',
# description='transform output pressure',
# temporal_domain = tdom,
# spatial_domain = sdom)
#
# ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
# outgoing_stream_l2_density_id,
# parameter_dictionary)
contactInfo = ContactInformation()
contactInfo.individual_names_given = "Bill"
contactInfo.individual_name_family = "Smith"
contactInfo.street_address = "111 First St"
contactInfo.city = "San Diego"
contactInfo.email = "*****@*****.**"
contactInfo.phones = ["858-555-6666"]
contactInfo.country = "USA"
contactInfo.postal_code = "92123"
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
contacts = [contactInfo],
iso_topic_category = "my_iso_topic_category_here",
quality_control_level = "1",
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L0 all data_process start")
try:
input_data_products = [ctd_parsed_data_product]
output_data_products = [ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id, ctd_l0_temperature_output_dp_id]
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L0_all_dprocdef_id,
in_data_product_ids = input_data_products,
out_data_product_ids = output_data_products
)
#activate only this data process just for coverage
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment)
log.debug("TestDataProductProvenance: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_conductivity_dprocdef_id,
in_data_product_ids = [ctd_l0_conductivity_output_dp_id],
out_data_product_ids = [ctd_l1_conductivity_output_dp_id])
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_pressure_dprocdef_id,
in_data_product_ids = [ctd_l0_pressure_output_dp_id],
out_data_product_ids = [ctd_l1_pressure_output_dp_id])
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_temperature_dprocdef_id,
in_data_product_ids = [ctd_l0_temperature_output_dp_id],
out_data_product_ids = [ctd_l1_temperature_output_dp_id])
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L2_salinity_dprocdef_id,
in_data_product_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id],
out_data_product_ids = [ctd_l2_salinity_output_dp_id])
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_Density data_process start")
try:
in_dp_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id]
out_dp_ids = [ctd_l2_density_output_dp_id]
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L2_density_dprocdef_id,
in_data_product_ids = in_dp_ids,
out_data_product_ids = out_dp_ids)
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('iaclient', name=ResourceAgentClient._get_agent_process_id(instDevice_id, process=FakeProcess())
# print 'activate_instrument: got ia client %s', self._ia_client
# log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id)
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the provenance info for the ctd density data product
#-------------------------------
provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id)
log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict))
#validate that products are represented
self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)])
density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id])
#-------------------------------
# Retrieve the extended resource for this data product
#-------------------------------
extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id)
self.assertEqual(1, len(extended_product.data_processes) )
self.assertEqual(3, len(extended_product.process_input_data_products) )
# log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list))
# log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes))
# log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products))
# log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value))
#-------------------------------
# Retrieve the extended resource for this data process
#-------------------------------
extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
# log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def))
# log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes))
# log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products))
self.assertEqual(1, len(extended_process_def.data_processes) )
self.assertEqual(3, len(extended_process_def.output_stream_definitions) )
self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process
#-------------------------------
# Request the xml report
#-------------------------------
results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id)
print results
#-------------------------------
# Cleanup
#-------------------------------
self.dpmsclient.delete_data_product(ctd_parsed_data_product)
self.dpmsclient.delete_data_product(log_data_product_id)
self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestObservatoryManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
#print 'TestObservatoryManagementServiceIntegration: started services'
@unittest.skip('this exists only for debugging the launch process')
def test_just_the_setup(self):
return
#@unittest.skip('targeting')
def test_resources_associations(self):
self._make_associations()
#@unittest.skip('targeting')
def test_find_related_frames_of_reference(self):
# finding subordinates gives a dict of obj lists, convert objs to ids
def idify(adict):
ids = {}
for k, v in adict.iteritems():
ids[k] = []
for obj in v:
ids[k].append(obj._id)
return ids
# a short version of the function we're testing, with id-ify
def short(resource_id, output_types):
ret = self.OMS.find_related_frames_of_reference(
resource_id, output_types)
return idify(ret)
#set up associations first
stuff = self._make_associations()
#basic traversal of tree from instrument to platform
ids = short(stuff.instrument_site_id, [RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
#since this is the first search, just make sure the input inst_id got stripped
if RT.InstrumentSite in ids:
self.assertNotIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
#basic traversal of tree from platform to instrument
ids = short(stuff.platform_siteb_id, [RT.InstrumentSite])
self.assertIn(RT.InstrumentSite, ids)
self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
self.assertNotIn(stuff.instrument_site2_id, ids[RT.InstrumentSite])
#full traversal of tree from observatory down to instrument
ids = short(stuff.observatory_id, [RT.InstrumentSite])
self.assertIn(RT.InstrumentSite, ids)
self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite])
#full traversal of tree from instrument to observatory
ids = short(stuff.instrument_site_id, [RT.Observatory])
self.assertIn(RT.Observatory, ids)
self.assertIn(stuff.observatory_id, ids[RT.Observatory])
#partial traversal, only down to platform
ids = short(stuff.observatory_id, [RT.Subsite, RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(RT.Subsite, ids)
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_site3_id, ids[RT.PlatformSite])
self.assertIn(stuff.subsite_id, ids[RT.Subsite])
self.assertIn(stuff.subsite2_id, ids[RT.Subsite])
self.assertIn(stuff.subsitez_id, ids[RT.Subsite])
self.assertIn(stuff.subsiteb_id, ids[RT.Subsite])
self.assertNotIn(RT.InstrumentSite, ids)
#partial traversal, only down to platform
ids = short(stuff.instrument_site_id, [RT.Subsite, RT.PlatformSite])
self.assertIn(RT.PlatformSite, ids)
self.assertIn(RT.Subsite, ids)
self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite])
self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite])
self.assertIn(stuff.subsite_id, ids[RT.Subsite])
self.assertIn(stuff.subsiteb_id, ids[RT.Subsite])
self.assertNotIn(stuff.subsite2_id, ids[RT.Subsite])
self.assertNotIn(stuff.subsitez_id, ids[RT.Subsite])
self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite])
self.assertNotIn(RT.Observatory, ids)
def _make_associations(self):
"""
create one of each resource and association used by OMS
to guard against problems in ion-definitions
"""
#raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41")
"""
the tree we're creating (observatory, sites, platforms, instruments)
rows are lettered, colums numbered.
- first row is implied a
- first column is implied 1
- site Z, just because
O--Sz
|
S--S2--P3--I4
|
Sb-Pb2-Ib3
|
P--I2
|
Pb
|
I
"""
#stuff we control
observatory_id, _ = self.RR.create(any_old(RT.Observatory))
subsite_id, _ = self.RR.create(any_old(RT.Subsite))
subsite2_id, _ = self.RR.create(any_old(RT.Subsite))
subsiteb_id, _ = self.RR.create(any_old(RT.Subsite))
subsitez_id, _ = self.RR.create(any_old(RT.Subsite))
platform_site_id, _ = self.RR.create(any_old(RT.PlatformSite))
platform_siteb_id, _ = self.RR.create(any_old(RT.PlatformSite))
platform_siteb2_id, _ = self.RR.create(any_old(RT.PlatformSite))
platform_site3_id, _ = self.RR.create(any_old(RT.PlatformSite))
instrument_site_id, _ = self.RR.create(any_old(RT.InstrumentSite))
instrument_site2_id, _ = self.RR.create(any_old(RT.InstrumentSite))
instrument_siteb3_id, _ = self.RR.create(any_old(RT.InstrumentSite))
instrument_site4_id, _ = self.RR.create(any_old(RT.InstrumentSite))
#stuff we associate to
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
deployment_id, _ = self.RR.create(any_old(RT.Deployment))
#observatory
self.RR.create_association(observatory_id, PRED.hasSite, subsite_id)
self.RR.create_association(observatory_id, PRED.hasSite, subsitez_id)
#site
self.RR.create_association(subsite_id, PRED.hasSite, subsite2_id)
self.RR.create_association(subsite_id, PRED.hasSite, subsiteb_id)
self.RR.create_association(subsite2_id, PRED.hasSite,
platform_site3_id)
self.RR.create_association(subsiteb_id, PRED.hasSite,
platform_siteb2_id)
self.RR.create_association(subsiteb_id, PRED.hasSite, platform_site_id)
#platform_site
self.RR.create_association(platform_site3_id, PRED.hasSite,
instrument_site4_id)
self.RR.create_association(platform_siteb2_id, PRED.hasSite,
instrument_siteb3_id)
self.RR.create_association(platform_site_id, PRED.hasSite,
instrument_site2_id)
self.RR.create_association(platform_site_id, PRED.hasSite,
platform_siteb_id)
self.RR.create_association(platform_siteb_id, PRED.hasSite,
instrument_site_id)
self.RR.create_association(platform_site_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_site_id, PRED.hasDevice,
platform_device_id)
self.RR.create_association(platform_site_id, PRED.hasDeployment,
deployment_id)
#instrument_site
self.RR.create_association(instrument_site_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(instrument_site_id, PRED.hasDevice,
instrument_device_id)
self.RR.create_association(instrument_site_id, PRED.hasDeployment,
deployment_id)
ret = DotDict()
ret.observatory_id = observatory_id
ret.subsite_id = subsite_id
ret.subsite2_id = subsite2_id
ret.subsiteb_id = subsiteb_id
ret.subsitez_id = subsitez_id
ret.platform_site_id = platform_site_id
ret.platform_siteb_id = platform_siteb_id
ret.platform_siteb2_id = platform_siteb2_id
ret.platform_site3_id = platform_site3_id
ret.instrument_site_id = instrument_site_id
ret.instrument_site2_id = instrument_site2_id
ret.instrument_siteb3_id = instrument_siteb3_id
ret.instrument_site4_id = instrument_site4_id
return ret
#@unittest.skip("targeting")
def test_create_observatory(self):
observatory_obj = IonObject(RT.Observatory,
name='TestFacility',
description='some new mf')
self.OMS.create_observatory(observatory_obj)
#@unittest.skip('targeting')
def test_find_observatory_org(self):
org_obj = IonObject(RT.Org,
name='TestOrg',
description='some new mf org')
org_id = self.OMS.create_marine_facility(org_obj)
observatory_obj = IonObject(RT.Observatory,
name='TestObservatory',
description='some new obs')
observatory_id = self.OMS.create_observatory(observatory_obj)
#make association
self.OMS.assign_resource_to_observatory_org(observatory_id, org_id)
#find association
org_objs = self.OMS.find_org_by_observatory(observatory_id)
self.assertEqual(1, len(org_objs))
self.assertEqual(org_id, org_objs[0]._id)
print("org_id=<" + org_id + ">")
#create a subsite with parent Observatory
subsite_obj = IonObject(RT.Subsite,
name='TestSubsite',
description='sample subsite')
subsite_id = self.OMS.create_subsite(subsite_obj, observatory_id)
self.assertIsNotNone(subsite_id, "Subsite not created.")
# verify that Subsite is linked to Observatory
mf_subsite_assoc = self.RR.get_association(observatory_id,
PRED.hasSite, subsite_id)
self.assertIsNotNone(mf_subsite_assoc,
"Subsite not connected to Observatory.")
# add the Subsite as a resource of this Observatory
self.OMS.assign_resource_to_observatory_org(resource_id=subsite_id,
org_id=org_id)
# verify that Subsite is linked to Org
org_subsite_assoc = self.RR.get_association(org_id, PRED.hasResource,
subsite_id)
self.assertIsNotNone(org_subsite_assoc,
"Subsite not connected as resource to Org.")
#create a logical platform with parent Subsite
platform_site_obj = IonObject(RT.PlatformSite,
name='TestPlatformSite',
description='sample logical platform')
platform_site_id = self.OMS.create_platform_site(
platform_site_obj, subsite_id)
self.assertIsNotNone(platform_site_id, "PlatformSite not created.")
# verify that PlatformSite is linked to Site
site_lp_assoc = self.RR.get_association(subsite_id, PRED.hasSite,
platform_site_id)
self.assertIsNotNone(site_lp_assoc,
"PlatformSite not connected to Site.")
# add the PlatformSite as a resource of this Observatory
self.OMS.assign_resource_to_observatory_org(
resource_id=platform_site_id, org_id=org_id)
# verify that PlatformSite is linked to Org
org_lp_assoc = self.RR.get_association(org_id, PRED.hasResource,
platform_site_id)
self.assertIsNotNone(org_lp_assoc,
"PlatformSite not connected as resource to Org.")
#create a logical instrument with parent logical platform
instrument_site_obj = IonObject(
RT.InstrumentSite,
name='TestInstrumentSite',
description='sample logical instrument')
instrument_site_id = self.OMS.create_instrument_site(
instrument_site_obj, platform_site_id)
self.assertIsNotNone(instrument_site_id, "InstrumentSite not created.")
# verify that InstrumentSite is linked to PlatformSite
li_lp_assoc = self.RR.get_association(platform_site_id, PRED.hasSite,
instrument_site_id)
self.assertIsNotNone(li_lp_assoc,
"InstrumentSite not connected to PlatformSite.")
# add the InstrumentSite as a resource of this Observatory
self.OMS.assign_resource_to_observatory_org(
resource_id=instrument_site_id, org_id=org_id)
# verify that InstrumentSite is linked to Org
org_li_assoc = self.RR.get_association(org_id, PRED.hasResource,
instrument_site_id)
self.assertIsNotNone(
org_li_assoc, "InstrumentSite not connected as resource to Org.")
# remove the InstrumentSite as a resource of this Observatory
self.OMS.unassign_resource_from_observatory_org(
instrument_site_id, org_id)
# verify that InstrumentSite is linked to Org
assocs, _ = self.RR.find_objects(org_id,
PRED.hasResource,
RT.InstrumentSite,
id_only=True)
self.assertEqual(len(assocs), 0)
# remove the InstrumentSite
self.OMS.delete_instrument_site(instrument_site_id)
assocs, _ = self.RR.find_objects(platform_site_id,
PRED.hasInstrument,
RT.InstrumentSite,
id_only=True)
self.assertEqual(len(assocs), 0)
# remove the PlatformSite as a resource of this Observatory
self.OMS.unassign_resource_from_observatory_org(
platform_site_id, org_id)
# verify that PlatformSite is linked to Org
assocs, _ = self.RR.find_objects(org_id,
PRED.hasResource,
RT.PlatformSite,
id_only=True)
self.assertEqual(len(assocs), 0)
# remove the Site as a resource of this Observatory
self.OMS.unassign_resource_from_observatory_org(subsite_id, org_id)
# verify that Site is linked to Org
assocs, _ = self.RR.find_objects(org_id,
PRED.hasResource,
RT.Subsite,
id_only=True)
self.assertEqual(len(assocs), 0)
