class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ datastore_name = CACHE_DATASTORE_NAME self.db = self.container.datastore_manager.get_datastore(datastore_name) self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.process_dispatcher.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) IngestionManagementIntTest.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Generic time-series data domain creation tdom, sdom = time_series_domain() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 200.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = 100.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 50.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = 100.0 #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) self.assertEquals(dp_obj.geospatial_point_center.lat, 150.0) log.debug('Created data product %s', dp_obj) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id2) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 300.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = 200.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 150.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = 200.0 # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description,'the very first dp') self.assertEquals(dp_obj.geospatial_point_center.lat, 250.0) log.debug('Updated data product %s', dp_obj) #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) self.assertEqual(dp_id, extended_product._id) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual(0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product(dp_id) # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) self.assertTrue(len(events) > 0) def test_data_product_stream_def(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id) self.assertEquals(ctd_stream_def_id, stream_def_id) def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) self.get_datastore(dataset_ids[0]) # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_ids[0]) self.assertIsInstance(replay_data, Granule) log.debug("The data retriever was able to replay the dataset that was attached to the data product " "we wanted to be persisted. Therefore the data product was indeed persisted with " "otherwise we could not have retrieved its dataset using the data retriever. Therefore " "this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'") data_product_object = self.rrclient.read(dp_id) self.assertEquals(data_product_object.name,'DP1') self.assertEquals(data_product_object.description,'some new dp') log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. " " Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the " "resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name, data_product_object.description)) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.rrclient.read(dp_id)
class TestObservatoryManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.org_management_service = OrgManagementServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #print 'TestObservatoryManagementServiceIntegration: started services' self.event_publisher = EventPublisher() # @unittest.skip('this exists only for debugging the launch process') # def test_just_the_setup(self): # return def destroy(self, resource_ids): self.OMS.force_delete_observatory(resource_ids.observatory_id) self.OMS.force_delete_subsite(resource_ids.subsite_id) self.OMS.force_delete_subsite(resource_ids.subsite2_id) self.OMS.force_delete_subsite(resource_ids.subsiteb_id) self.OMS.force_delete_subsite(resource_ids.subsitez_id) self.OMS.force_delete_platform_site(resource_ids.platform_site_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb2_id) self.OMS.force_delete_platform_site(resource_ids.platform_site3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site2_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_siteb3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site4_id) #@unittest.skip('targeting') def test_observatory_management(self): resources = self._make_associations() self._do_test_find_related_sites(resources) self._do_test_get_sites_devices_status(resources) self._do_test_find_site_data_products(resources) self._do_test_find_related_frames_of_reference(resources) self._do_test_create_geospatial_point_center(resources) self._do_test_find_observatory_org(resources) self.destroy(resources) def _do_test_find_related_sites(self, resources): site_resources, site_children = self.OMS.find_related_sites(resources.org_id) #import sys, pprint #print >> sys.stderr, pprint.pformat(site_resources) #print >> sys.stderr, pprint.pformat(site_children) #self.assertIn(resources.org_id, site_resources) self.assertIn(resources.observatory_id, site_resources) self.assertIn(resources.subsite_id, site_resources) self.assertIn(resources.subsite_id, site_resources) self.assertIn(resources.subsite2_id, site_resources) self.assertIn(resources.platform_site_id, site_resources) self.assertIn(resources.instrument_site_id, site_resources) self.assertEquals(len(site_resources), 13) self.assertEquals(site_resources[resources.observatory_id].type_, RT.Observatory) self.assertIn(resources.org_id, site_children) self.assertIn(resources.observatory_id, site_children) self.assertIn(resources.subsite_id, site_children) self.assertIn(resources.subsite_id, site_children) self.assertIn(resources.subsite2_id, site_children) self.assertIn(resources.platform_site_id, site_children) self.assertNotIn(resources.instrument_site_id, site_children) self.assertEquals(len(site_children), 9) self.assertIsInstance(site_children[resources.subsite_id], list) self.assertEquals(len(site_children[resources.subsite_id]), 2) def _do_test_get_sites_devices_status(self, resources): result_dict = self.OMS.get_sites_devices_status(resources.org_id) site_resources = result_dict.get("site_resources", None) site_children = result_dict.get("site_children", None) self.assertEquals(len(site_resources), 14) self.assertEquals(len(site_children), 9) result_dict = self.OMS.get_sites_devices_status(resources.org_id, include_devices=True, include_status=True) log.debug("RESULT DICT: %s", result_dict.keys()) site_resources = result_dict.get("site_resources", None) site_children = result_dict.get("site_children", None) site_status = result_dict.get("site_status", None) self.assertEquals(len(site_resources), 14) self.assertEquals(len(site_children), 9) result_dict = self.OMS.get_sites_devices_status(resources.observatory_id, include_devices=True, include_status=True) site_resources = result_dict.get("site_resources") site_children = result_dict.get("site_children") site_status = result_dict.get("site_status") self.assertEquals(len(site_resources), 13) self.assertEquals(len(site_children), 8) def _do_test_find_site_data_products(self, resources): res_dict = self.OMS.find_site_data_products(resources.org_id) #import sys, pprint #print >> sys.stderr, pprint.pformat(res_dict) self.assertIsNone(res_dict['data_product_resources']) self.assertIn(resources.platform_device_id, res_dict['device_data_products']) self.assertIn(resources.instrument_device_id, res_dict['device_data_products']) #@unittest.skip('targeting') def _do_test_find_related_frames_of_reference(self, stuff): # finding subordinates gives a dict of obj lists, convert objs to ids def idify(adict): ids = {} for k, v in adict.iteritems(): ids[k] = [] for obj in v: ids[k].append(obj._id) return ids # a short version of the function we're testing, with id-ify def short(resource_id, output_types): ret = self.OMS.find_related_frames_of_reference(resource_id, output_types) return idify(ret) #set up associations first stuff = self._make_associations() #basic traversal of tree from instrument to platform ids = short(stuff.instrument_site_id, [RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) #since this is the first search, just make sure the input inst_id got stripped if RT.InstrumentSite in ids: self.assertNotIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #basic traversal of tree from platform to instrument ids = short(stuff.platform_siteb_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) self.assertNotIn(stuff.instrument_site2_id, ids[RT.InstrumentSite]) #full traversal of tree from observatory down to instrument ids = short(stuff.observatory_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #full traversal of tree from instrument to observatory ids = short(stuff.instrument_site_id, [RT.Observatory]) self.assertIn(RT.Observatory, ids) self.assertIn(stuff.observatory_id, ids[RT.Observatory]) #partial traversal, only down to platform ids = short(stuff.observatory_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site3_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(RT.InstrumentSite, ids) #partial traversal, only down to platform ids = short(stuff.instrument_site_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertNotIn(RT.Observatory, ids) self.destroy(stuff) def _make_associations(self): """ create one of each resource and association used by OMS to guard against problems in ion-definitions """ #raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41") """ the tree we're creating (observatory, sites, platforms, instruments) rows are lettered, colums numbered. - first row is implied a - first column is implied 1 - site Z, just because O--Sz | S--S2--P3--I4 | Sb-Pb2-Ib3 | P--I2 <- PlatformDevice, InstrumentDevice2 | Pb <- PlatformDevice b | I <- InstrumentDevice """ org_id = self.OMS.create_marine_facility(any_old(RT.Org)) def create_under_org(resource_type, extra_fields=None): obj = any_old(resource_type, extra_fields) if RT.InstrumentDevice == resource_type: resource_id = self.IMS.create_instrument_device(obj) else: resource_id, _ = self.RR.create(obj) self.OMS.assign_resource_to_observatory_org(resource_id=resource_id, org_id=org_id) return resource_id #stuff we control observatory_id = create_under_org(RT.Observatory) subsite_id = create_under_org(RT.Subsite) subsite2_id = create_under_org(RT.Subsite) subsiteb_id = create_under_org(RT.Subsite) subsitez_id = create_under_org(RT.Subsite) platform_site_id = create_under_org(RT.PlatformSite) platform_siteb_id = create_under_org(RT.PlatformSite) platform_siteb2_id = create_under_org(RT.PlatformSite) platform_site3_id = create_under_org(RT.PlatformSite) instrument_site_id = create_under_org(RT.InstrumentSite) instrument_site2_id = create_under_org(RT.InstrumentSite) instrument_siteb3_id = create_under_org(RT.InstrumentSite) instrument_site4_id = create_under_org(RT.InstrumentSite) #stuff we associate to instrument_device_id = create_under_org(RT.InstrumentDevice) instrument_device2_id = create_under_org(RT.InstrumentDevice) platform_device_id = create_under_org(RT.PlatformDevice) platform_deviceb_id = create_under_org(RT.PlatformDevice) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) deployment_id, _ = self.RR.create(any_old(RT.Deployment)) #observatory self.RR.create_association(observatory_id, PRED.hasSite, subsite_id) self.RR.create_association(observatory_id, PRED.hasSite, subsitez_id) #site self.RR.create_association(subsite_id, PRED.hasSite, subsite2_id) self.RR.create_association(subsite_id, PRED.hasSite, subsiteb_id) self.RR.create_association(subsite2_id, PRED.hasSite, platform_site3_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_siteb2_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_site_id) #platform_site(s) self.RR.create_association(platform_site3_id, PRED.hasSite, instrument_site4_id) self.RR.create_association(platform_siteb2_id, PRED.hasSite, instrument_siteb3_id) self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site2_id) self.RR.create_association(platform_site_id, PRED.hasSite, platform_siteb_id) self.RR.create_association(platform_siteb_id, PRED.hasSite, instrument_site_id) self.RR.create_association(platform_siteb_id, PRED.hasDevice, platform_deviceb_id) #test network parent link self.OMS.assign_device_to_network_parent(platform_device_id, platform_deviceb_id) self.RR.create_association(platform_site_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id) self.RR.create_association(platform_site_id, PRED.hasDeployment, deployment_id) #instrument_site(s) self.RR.create_association(instrument_site_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(instrument_site_id, PRED.hasDeployment, deployment_id) self.RR.create_association(instrument_site2_id, PRED.hasDevice, instrument_device2_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id) ret = DotDict() ret.org_id = org_id ret.observatory_id = observatory_id ret.subsite_id = subsite_id ret.subsite2_id = subsite2_id ret.subsiteb_id = subsiteb_id ret.subsitez_id = subsitez_id ret.platform_site_id = platform_site_id ret.platform_siteb_id = platform_siteb_id ret.platform_siteb2_id = platform_siteb2_id ret.platform_site3_id = platform_site3_id ret.instrument_site_id = instrument_site_id ret.instrument_site2_id = instrument_site2_id ret.instrument_siteb3_id = instrument_siteb3_id ret.instrument_site4_id = instrument_site4_id ret.instrument_device_id = instrument_device_id ret.instrument_device2_id = instrument_device2_id ret.platform_device_id = platform_device_id ret.platform_deviceb_id = platform_deviceb_id ret.instrument_model_id = instrument_model_id ret.platform_model_id = platform_model_id ret.deployment_id = deployment_id return ret #@unittest.skip("targeting") def test_create_observatory(self): observatory_obj = IonObject(RT.Observatory, name='TestFacility', description='some new mf') observatory_id = self.OMS.create_observatory(observatory_obj) self.OMS.force_delete_observatory(observatory_id) #@unittest.skip("targeting") def _do_test_create_geospatial_point_center(self, resources): platformsite_obj = IonObject(RT.PlatformSite, name='TestPlatformSite', description='some new TestPlatformSite') geo_index_obj = IonObject(OT.GeospatialBounds) geo_index_obj.geospatial_latitude_limit_north = 20.0 geo_index_obj.geospatial_latitude_limit_south = 10.0 geo_index_obj.geospatial_longitude_limit_east = 15.0 geo_index_obj.geospatial_longitude_limit_west = 20.0 platformsite_obj.constraint_list = [geo_index_obj] platformsite_id = self.OMS.create_platform_site(platformsite_obj) # now get the dp back to see if it was updated platformsite_obj = self.OMS.read_platform_site(platformsite_id) self.assertEquals('some new TestPlatformSite', platformsite_obj.description) self.assertAlmostEqual(15.0, platformsite_obj.geospatial_point_center.lat, places=1) #now adjust a few params platformsite_obj.description = 'some old TestPlatformSite' geo_index_obj = IonObject(OT.GeospatialBounds) geo_index_obj.geospatial_latitude_limit_north = 30.0 geo_index_obj.geospatial_latitude_limit_south = 20.0 platformsite_obj.constraint_list = [geo_index_obj] update_result = self.OMS.update_platform_site(platformsite_obj) # now get the dp back to see if it was updated platformsite_obj = self.OMS.read_platform_site(platformsite_id) self.assertEquals('some old TestPlatformSite', platformsite_obj.description) self.assertAlmostEqual(25.0, platformsite_obj.geospatial_point_center.lat, places=1) self.OMS.force_delete_platform_site(platformsite_id) #@unittest.skip("targeting") def _do_test_find_observatory_org(self, resources): log.debug("Make TestOrg") org_obj = IonObject(RT.Org, name='TestOrg', description='some new mf org') org_id = self.OMS.create_marine_facility(org_obj) log.debug("Make Observatory") observatory_obj = IonObject(RT.Observatory, name='TestObservatory', description='some new obs') observatory_id = self.OMS.create_observatory(observatory_obj) log.debug("assign observatory to org") self.OMS.assign_resource_to_observatory_org(observatory_id, org_id) log.debug("verify assigment") org_objs = self.OMS.find_org_by_observatory(observatory_id) self.assertEqual(1, len(org_objs)) self.assertEqual(org_id, org_objs[0]._id) log.debug("org_id=<" + org_id + ">") log.debug("create a subsite with parent Observatory") subsite_obj = IonObject(RT.Subsite, name= 'TestSubsite', description = 'sample subsite') subsite_id = self.OMS.create_subsite(subsite_obj, observatory_id) self.assertIsNotNone(subsite_id, "Subsite not created.") log.debug("verify that Subsite is linked to Observatory") mf_subsite_assoc = self.RR.get_association(observatory_id, PRED.hasSite, subsite_id) self.assertIsNotNone(mf_subsite_assoc, "Subsite not connected to Observatory.") log.debug("add the Subsite as a resource of this Observatory") self.OMS.assign_resource_to_observatory_org(resource_id=subsite_id, org_id=org_id) log.debug("verify that Subsite is linked to Org") org_subsite_assoc = self.RR.get_association(org_id, PRED.hasResource, subsite_id) self.assertIsNotNone(org_subsite_assoc, "Subsite not connected as resource to Org.") log.debug("create a logical platform with parent Subsite") platform_site_obj = IonObject(RT.PlatformSite, name= 'TestPlatformSite', description = 'sample logical platform') platform_site_id = self.OMS.create_platform_site(platform_site_obj, subsite_id) self.assertIsNotNone(platform_site_id, "PlatformSite not created.") log.debug("verify that PlatformSite is linked to Site") site_lp_assoc = self.RR.get_association(subsite_id, PRED.hasSite, platform_site_id) self.assertIsNotNone(site_lp_assoc, "PlatformSite not connected to Site.") log.debug("add the PlatformSite as a resource of this Observatory") self.OMS.assign_resource_to_observatory_org(resource_id=platform_site_id, org_id=org_id) log.debug("verify that PlatformSite is linked to Org") org_lp_assoc = self.RR.get_association(org_id, PRED.hasResource, platform_site_id) self.assertIsNotNone(org_lp_assoc, "PlatformSite not connected as resource to Org.") log.debug("create a logical instrument with parent logical platform") instrument_site_obj = IonObject(RT.InstrumentSite, name= 'TestInstrumentSite', description = 'sample logical instrument') instrument_site_id = self.OMS.create_instrument_site(instrument_site_obj, platform_site_id) self.assertIsNotNone(instrument_site_id, "InstrumentSite not created.") log.debug("verify that InstrumentSite is linked to PlatformSite") li_lp_assoc = self.RR.get_association(platform_site_id, PRED.hasSite, instrument_site_id) self.assertIsNotNone(li_lp_assoc, "InstrumentSite not connected to PlatformSite.") log.debug("add the InstrumentSite as a resource of this Observatory") self.OMS.assign_resource_to_observatory_org(resource_id=instrument_site_id, org_id=org_id) log.debug("verify that InstrumentSite is linked to Org") org_li_assoc = self.RR.get_association(org_id, PRED.hasResource, instrument_site_id) self.assertIsNotNone(org_li_assoc, "InstrumentSite not connected as resource to Org.") log.debug("remove the InstrumentSite as a resource of this Observatory") self.OMS.unassign_resource_from_observatory_org(instrument_site_id, org_id) log.debug("verify that InstrumentSite is linked to Org") assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.InstrumentSite, id_only=True ) self.assertEqual(0, len(assocs)) log.debug("remove the InstrumentSite, association should drop automatically") self.OMS.delete_instrument_site(instrument_site_id) assocs, _ = self.RR.find_objects(platform_site_id, PRED.hasSite, RT.InstrumentSite, id_only=True ) self.assertEqual(0, len(assocs)) log.debug("remove the PlatformSite as a resource of this Observatory") self.OMS.unassign_resource_from_observatory_org(platform_site_id, org_id) log.debug("verify that PlatformSite is linked to Org") assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.PlatformSite, id_only=True ) self.assertEqual(0, len(assocs)) log.debug("remove the Site as a resource of this Observatory") self.OMS.unassign_resource_from_observatory_org(subsite_id, org_id) log.debug("verify that Site is linked to Org") assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.Subsite, id_only=True ) self.assertEqual(0, len(assocs)) self.RR.delete(org_id) self.OMS.force_delete_observatory(observatory_id) self.OMS.force_delete_subsite(subsite_id) self.OMS.force_delete_platform_site(platform_site_id) self.OMS.force_delete_instrument_site(instrument_site_id) @attr('EXT') def test_observatory_extensions(self): obs_id = self.RR2.create(any_old(RT.Observatory)) pss_id = self.RR2.create(any_old(RT.PlatformSite, dict(alt_resource_type="StationSite"))) pas_id = self.RR2.create(any_old(RT.PlatformSite, dict(alt_resource_type="PlatformAssemblySite"))) pcs_id = self.RR2.create(any_old(RT.PlatformSite, dict(alt_resource_type="PlatformComponentSite"))) ins_id = self.RR2.create(any_old(RT.InstrumentSite)) obs_obj = self.RR2.read(obs_id) pss_obj = self.RR2.read(pss_id) pas_obj = self.RR2.read(pas_id) pcs_obj = self.RR2.read(pcs_id) ins_obj = self.RR2.read(ins_id) self.RR2.create_association(obs_id, PRED.hasSite, pss_id) self.RR2.create_association(pss_id, PRED.hasSite, pas_id) self.RR2.create_association(pas_id, PRED.hasSite, pcs_id) self.RR2.create_association(pcs_id, PRED.hasSite, ins_id) extended_obs = self.OMS.get_observatory_site_extension(obs_id, user_id=12345) self.assertEqual([pss_obj], extended_obs.computed.platform_station_sites.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_obs.computed.platform_station_sites.status) self.assertEqual([pas_obj], extended_obs.computed.platform_assembly_sites.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_obs.computed.platform_assembly_sites.status) self.assertEqual([pcs_obj], extended_obs.computed.platform_component_sites.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_obs.computed.platform_component_sites.status) self.assertEqual([ins_obj], extended_obs.computed.instrument_sites.value) extended_pss = self.OMS.get_observatory_site_extension(obs_id, user_id=12345) self.assertEqual([pas_obj], extended_pss.computed.platform_assembly_sites.value) self.assertEqual([pcs_obj], extended_pss.computed.platform_component_sites.value) self.assertEqual([ins_obj], extended_pss.computed.instrument_sites.value) extended_pas = self.OMS.get_observatory_site_extension(pas_id, user_id=12345) self.assertEqual([pcs_obj], extended_pas.computed.platform_component_sites.value) self.assertEqual([ins_obj], extended_pas.computed.instrument_sites.value) extended_pcs = self.OMS.get_platform_component_site_extension(pcs_id, user_id=12345) self.assertEqual([ins_obj], extended_pcs.computed.instrument_sites.value) #@unittest.skip("in development...") @attr('EXT') @attr('EXT1') def test_observatory_org_extended(self): stuff = self._make_associations() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.damsclient.assign_data_product(input_resource_id=stuff.instrument_device_id, data_product_id=data_product_id1) #Create a user to be used as regular member member_actor_obj = IonObject(RT.ActorIdentity, name='org member actor') member_actor_id,_ = self.RR.create(member_actor_obj) assert(member_actor_id) member_actor_header = get_actor_header(member_actor_id) member_user_obj = IonObject(RT.UserInfo, name='org member user') member_user_id,_ = self.RR.create(member_user_obj) assert(member_user_id) self.RR.create_association(subject=member_actor_id, predicate=PRED.hasInfo, object=member_user_id) #Build the Service Agreement Proposal to enroll a user actor sap = IonObject(OT.EnrollmentProposal,consumer=member_actor_id, provider=stuff.org_id ) sap_response = self.org_management_service.negotiate(sap, headers=member_actor_header ) #enroll the member without using negotiation self.org_management_service.enroll_member(org_id=stuff.org_id, actor_id=member_actor_id) #-------------------------------------------------------------------------------- # Get the extended Site (platformSite) #-------------------------------------------------------------------------------- try: extended_site = self.OMS.get_site_extension(stuff.platform_site_id) except: log.error('failed to get extended site', exc_info=True) raise log.debug("extended_site: %r ", extended_site) self.assertEqual(1, len(extended_site.platform_devices)) self.assertEqual(1, len(extended_site.platform_models)) self.assertEqual(stuff.platform_device_id, extended_site.platform_devices[0]._id) self.assertEqual(stuff.platform_model_id, extended_site.platform_models[0]._id) log.debug("verify that PlatformDeviceb is linked to PlatformDevice with hasNetworkParent link") associations = self.RR.find_associations(subject=stuff.platform_deviceb_id, predicate=PRED.hasNetworkParent, object=stuff.platform_device_id, id_only=True) self.assertIsNotNone(associations, "PlatformDevice child not connected to PlatformDevice parent.") #-------------------------------------------------------------------------------- # Get the extended Org #-------------------------------------------------------------------------------- #test the extended resource extended_org = self.OMS.get_marine_facility_extension(stuff.org_id) log.debug("test_observatory_org_extended: extended_org: %s ", str(extended_org)) #self.assertEqual(2, len(extended_org.instruments_deployed) ) #self.assertEqual(1, len(extended_org.platforms_not_deployed) ) self.assertEqual(2, extended_org.number_of_platforms) self.assertEqual(2, len(extended_org.platform_models) ) self.assertEqual(2, extended_org.number_of_instruments) self.assertEqual(2, len(extended_org.instrument_models) ) self.assertEqual(1, len(extended_org.members)) self.assertNotEqual(extended_org.members[0]._id, member_actor_id) self.assertEqual(extended_org.members[0]._id, member_user_id) self.assertEqual(1, len(extended_org.open_requests)) self.assertTrue(len(extended_site.deployments)>0) self.assertEqual(len(extended_site.deployments), len(extended_site.deployment_info)) #test the extended resource of the ION org ion_org_id = self.org_management_service.find_org() extended_org = self.OMS.get_marine_facility_extension(ion_org_id._id, user_id=12345) log.debug("test_observatory_org_extended: extended_ION_org: %s ", str(extended_org)) self.assertEqual(1, len(extended_org.members)) self.assertEqual(0, extended_org.number_of_platforms) #self.assertEqual(1, len(extended_org.sites)) #-------------------------------------------------------------------------------- # Get the extended Site #-------------------------------------------------------------------------------- #create device state events to use for op /non-op filtering in extended t = get_ion_ts() self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent', origin = stuff.instrument_device_id, state=ResourceAgentState.STREAMING ) self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent', origin = stuff.instrument_device2_id, state=ResourceAgentState.INACTIVE ) extended_site = self.OMS.get_site_extension(stuff.instrument_site2_id) log.debug("test_observatory_org_extended: extended_site: %s ", str(extended_site)) self.dpclient.delete_data_product(data_product_id1)
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateInstrumentIntegration, self).setUp() config = DotDict() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.usernotificationclient = UserNotificationServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_notification(self, user_name = '', instrument_id='', product_id=''): #-------------------------------------------------------------------------------------- # Make notification request objects #-------------------------------------------------------------------------------------- notification_request_1 = NotificationRequest( name= 'notification_1', origin=instrument_id, origin_type="instrument", event_type='ResourceLifecycleEvent') notification_request_2 = NotificationRequest( name='notification_2', origin=product_id, origin_type="data product", event_type='DetectionEvent') #-------------------------------------------------------------------------------------- # Create a user and get the user_id #-------------------------------------------------------------------------------------- user = UserInfo() user.name = user_name user.contact.email = '*****@*****.**' % user_name user_id, _ = self.rrclient.create(user) #-------------------------------------------------------------------------------------- # Create notification #-------------------------------------------------------------------------------------- self.usernotificationclient.create_notification(notification=notification_request_1, user_id=user_id) self.usernotificationclient.create_notification(notification=notification_request_2, user_id=user_id) log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id) ) return user_id def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def _check_computed_attributes_of_extended_instrument(self, expected_instrument_device_id = '',extended_instrument = None): # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) # the following assert will not work without elasticsearch. #self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) # Verify the computed attribute for user notification requests self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) notifications = extended_instrument.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(expected_instrument_device_id, notification.origin) self.assertEqual("instrument", notification.origin_type) self.assertEqual('ResourceLifecycleEvent', notification.event_type) def _check_computed_attributes_of_extended_product(self, expected_data_product_id = '', extended_data_product = None): self.assertEqual(expected_data_product_id, extended_data_product._id) log.debug("extended_data_product.computed: %s", extended_data_product.computed) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_data_product.computed.product_download_size_estimated, ComputedFloatValue) self.assertIsInstance(extended_data_product.computed.number_active_subscriptions, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue) self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue) self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.user_notification_requests, ComputedListValue) self.assertIsInstance(extended_data_product.computed.active_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.past_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_contents_updated, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue) # exact text here keeps changing to fit UI capabilities. keep assertion general... self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value) ) notifications = extended_data_product.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(expected_data_product_id, notification.origin) self.assertEqual("data product", notification.origin_type) self.assertEqual('DetectionEvent', notification.event_type) @attr('LOCOINT') #@unittest.skip('refactoring') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 90}}}) def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw') parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) " , instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config, alerts= []) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s' , data_product_id1) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s' , dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # setup notifications for the device and parsed data product user_id_1 = self._create_notification( user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1) #---------- Create notifications for another user and verify that we see different computed subscriptions for the two users --------- user_id_2 = self._create_notification( user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('Data product streams2 = %s' , str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id2 = %s' , dataset_ids[0]) self.raw_dataset = dataset_ids[0] def start_instrument_agent(): self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) #log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s" , str(self._ia_client)) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s" , str(retval)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.INACTIVE, state) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s" , str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.IDLE, state) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s" , str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s" , str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.STOPPED, state) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.IDLE, state) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) for i in xrange(10): monitor = DatasetMonitor(dataset_id=self.parsed_dataset) self._ia_client.execute_resource(AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)) if not monitor.wait(): raise AssertionError('Failed on the %ith granule' % i) monitor.stop() # cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) # for i in xrange(10): # retval = self._ia_client.execute_resource(cmd) # log.debug("test_activateInstrumentSample: return from sample %s" , str(retval)) log.debug( "test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s" , str(reply)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data_raw, Granule) rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw) log.debug("RDT raw: %s", str(rdt_raw.pretty_print()) ) self.assertIn('raw', rdt_raw) raw_vals = rdt_raw['raw'] all_raw = "".join(raw_vals) # look for 't' entered after a prompt -- ">t" t_commands = all_raw.count(">t") if 10 != t_commands: log.error("%s raw_vals: ", len(raw_vals)) for i, r in enumerate(raw_vals): log.error("raw val %s: %s", i, [r]) self.fail("Expected 10 't' strings in raw_vals, got %s" % t_commands) else: log.debug("%s raw_vals: ", len(raw_vals)) for i, r in enumerate(raw_vals): log.debug("raw val %s: %s", i, [r]) replay_data_parsed = self.dataretrieverclient.retrieve(self.parsed_dataset) self.assertIsInstance(replay_data_parsed, Granule) rdt_parsed = RecordDictionaryTool.load_from_granule(replay_data_parsed) log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt_parsed.pretty_print()) ) self.assertIn('temp', rdt_parsed) temp_vals = rdt_parsed['temp'] pressure_vals = rdt_parsed['pressure'] if 10 != len(temp_vals): log.error("%s temp_vals: %s", len(temp_vals), temp_vals) self.fail("Expected 10 temp_vals, got %s" % len(temp_vals)) log.debug("l4-ci-sa-rq-138") """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ #-------------------------------------------------------------------------------- # Get the extended data product to see if it contains the granules #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id1, user_id=user_id_1) def poller(extended_product): return len(extended_product.computed.user_notification_requests.value) == 1 poll(poller, extended_product, timeout=30) self._check_computed_attributes_of_extended_product( expected_data_product_id = data_product_id1, extended_data_product = extended_product) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_1) #-------------------------------------------------------------------------------- # For the second user, check the extended data product and the extended intrument #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id2, user_id=user_id_2) self._check_computed_attributes_of_extended_product(expected_data_product_id = data_product_id2, extended_data_product = extended_product) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_2) self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument) #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(data_product_id1) self.dpclient.delete_data_product(data_product_id2)
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateInstrumentIntegration, self).setUp() config = DotDict() config.bootstrap.use_es = True self._start_container() self.addCleanup(TestActivateInstrumentIntegration.es_cleanup) self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.usernotificationclient = UserNotificationServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() @staticmethod def es_cleanup(): es_host = CFG.get_safe('server.elasticsearch.host', 'localhost') es_port = CFG.get_safe('server.elasticsearch.port', '9200') es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10) indexes = STD_INDEXES.keys() indexes.append('%s_resources_index' % get_sys_name().lower()) indexes.append('%s_events_index' % get_sys_name().lower()) for index in indexes: IndexManagementService._es_call(es.river_couchdb_delete, index) IndexManagementService._es_call(es.index_delete, index) def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_notification(self, user_name='', instrument_id='', product_id=''): #-------------------------------------------------------------------------------------- # Make notification request objects #-------------------------------------------------------------------------------------- notification_request_1 = NotificationRequest( name='notification_1', origin=instrument_id, origin_type="instrument", event_type='ResourceLifecycleEvent') notification_request_2 = NotificationRequest( name='notification_2', origin=product_id, origin_type="data product", event_type='DetectionEvent') #-------------------------------------------------------------------------------------- # Create a user and get the user_id #-------------------------------------------------------------------------------------- user = UserInfo() user.name = user_name user.contact.email = '*****@*****.**' % user_name user_id, _ = self.rrclient.create(user) #-------------------------------------------------------------------------------------- # Create notification #-------------------------------------------------------------------------------------- self.usernotificationclient.create_notification( notification=notification_request_1, user_id=user_id) self.usernotificationclient.create_notification( notification=notification_request_2, user_id=user_id) log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id)) return user_id def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def _check_computed_attributes_of_extended_instrument( self, expected_instrument_device_id='', extended_instrument=None): # Verify that computed attributes exist for the extended instrument self.assertIsInstance( extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance( extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.location_status_roll_up, ComputedIntValue) # the following assert will not work without elasticsearch. #self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) # Verify the computed attribute for user notification requests self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value)) notifications = extended_instrument.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(expected_instrument_device_id, notification.origin) self.assertEqual("instrument", notification.origin_type) self.assertEqual('ResourceLifecycleEvent', notification.event_type) def _check_computed_attributes_of_extended_product( self, expected_data_product_id='', extended_data_product=None): self.assertEqual(expected_data_product_id, extended_data_product._id) log.debug("extended_data_product.computed: %s", extended_data_product.computed) # Verify that computed attributes exist for the extended instrument self.assertIsInstance( extended_data_product.computed.product_download_size_estimated, ComputedFloatValue) self.assertIsInstance( extended_data_product.computed.number_active_subscriptions, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue) self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue) self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue) self.assertIsInstance( extended_data_product.computed.user_notification_requests, ComputedListValue) self.assertIsInstance( extended_data_product.computed.active_user_subscriptions, ComputedListValue) self.assertIsInstance( extended_data_product.computed.past_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue) self.assertIsInstance( extended_data_product.computed.data_contents_updated, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue) # exact text here keeps changing to fit UI capabilities. keep assertion general... self.assertIn( 'ok', extended_data_product.computed.last_granule.value['quality_flag']) self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value)) notifications = extended_data_product.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(expected_data_product_id, notification.origin) self.assertEqual("data product", notification.origin_type) self.assertEqual('DetectionEvent', notification.event_type) @attr('LOCOINT') #@unittest.skip('refactoring') @unittest.skipIf(not use_es, 'No ElasticSearch') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint': {'receive': {'timeout': 90}}}) def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ' ) instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config, alerts=[]) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'raw', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) # setup notifications for the device and parsed data product user_id_1 = self._create_notification(user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1) #---------- Create notifications for another user and verify that we see different computed subscriptions for the two users --------- user_id_2 = self._create_notification(user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('Data product streams2 = %s', str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id2 = %s', dataset_ids[0]) self.raw_dataset = dataset_ids[0] #elastic search debug es_indexes, _ = self.container.resource_registry.find_resources( restype='ElasticSearchIndex') log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes]) log.debug('Bootstrap %s', CFG.bootstrap.use_es) def start_instrument_agent(): self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) #log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s", str(retval)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.INACTIVE, state) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.IDLE, state) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.STOPPED, state) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.IDLE, state) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) for i in xrange(10): retval = self._ia_client.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from sample %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data_raw, Granule) rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw) log.debug("RDT raw: %s", str(rdt_raw.pretty_print())) self.assertIn('raw', rdt_raw) raw_vals = rdt_raw['raw'] all_raw = "".join(raw_vals) # look for 't' entered after a prompt -- ">t" t_commands = all_raw.count(">t") if 10 != t_commands: log.error("%s raw_vals: ", len(raw_vals)) for i, r in enumerate(raw_vals): log.error("raw val %s: %s", i, [r]) self.fail("Expected 10 't' strings in raw_vals, got %s" % t_commands) else: log.debug("%s raw_vals: ", len(raw_vals)) for i, r in enumerate(raw_vals): log.debug("raw val %s: %s", i, [r]) replay_data_parsed = self.dataretrieverclient.retrieve( self.parsed_dataset) self.assertIsInstance(replay_data_parsed, Granule) rdt_parsed = RecordDictionaryTool.load_from_granule(replay_data_parsed) log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt_parsed.pretty_print())) self.assertIn('temp', rdt_parsed) temp_vals = rdt_parsed['temp'] pressure_vals = rdt_parsed['pressure'] if 10 != len(temp_vals): log.error("%s temp_vals: %s", len(temp_vals), temp_vals) self.fail("Expected 10 temp_vals, got %s" % len(temp_vals)) log.debug("l4-ci-sa-rq-138") """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ #-------------------------------------------------------------------------------- # Get the extended data product to see if it contains the granules #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id1, user_id=user_id_1) def poller(extended_product): return len(extended_product.computed.user_notification_requests. value) == 1 poll(poller, extended_product, timeout=30) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id1, extended_data_product=extended_product) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_1) #-------------------------------------------------------------------------------- # For the second user, check the extended data product and the extended intrument #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id2, user_id=user_id_2) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id2, extended_data_product=extended_product) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_2) self._check_computed_attributes_of_extended_instrument( expected_instrument_device_id=instDevice_id, extended_instrument=extended_instrument) #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(data_product_id1) self.dpclient.delete_data_product(data_product_id2)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ datastore_name = CACHE_DATASTORE_NAME self.db = self.container.datastore_manager.get_datastore(datastore_name) self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config) def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore @unittest.skip('OBE') def test_get_last_update(self): # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 1d spatial topology (station/trajectory) sdom = sdom.dump() tdom = tdom.dump() #@TODO: DO NOT DO THIS, WHEN THIS TEST IS REWRITTEN GET RID OF THIS, IT WILL FAIL, thanks -Luke parameter_dictionary = get_param_dict('ctd_parsed_param_dict') parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) data_product_id = self.dpsc_cli.create_data_product(data_product=dp_obj, stream_definition_id=self.stream_def_id, parameter_dictionary=parameter_dictionary) stream_ids, garbage = self.rrclient.find_objects(data_product_id, PRED.hasStream, id_only=True) stream_id = stream_ids[0] fake_lu = LastUpdate() fake_lu_doc = self.db._ion_object_to_persistence_dict(fake_lu) self.db.create_doc(fake_lu_doc, object_id=stream_id) #------------------------------------------ # Now execute #------------------------------------------ res = self.dpsc_cli.get_last_update(data_product_id=data_product_id) self.assertTrue(isinstance(res[stream_id], LastUpdate), 'retrieving documents failed') def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ log.debug('test_createDataProduct: Creating new data product w/o a stream definition (L4-CI-SA-RQ-308)') # Generic time-series data domain creation tdom, sdom = time_series_domain() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) log.debug('new dp_id = %s' % dp_id) log.debug("test_createDataProduct: Data product info from registry %s (L4-CI-SA-RQ-308)", str(dp_obj)) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description,'the very first dp') #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) self.assertEqual(dp_id, extended_product._id) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual(0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object try: dp_obj = self.dpsc_cli.read_data_product(dp_id) except NotFound as ex: pass else: self.fail("force deleted data product was found during read") # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) # now try to get the deleted dp object #todo: the RR should perhaps not return retired data products # dp_obj = self.dpsc_cli.read_data_product(dp_id) # now try to delete the already deleted dp object # log.debug( "deleting non-existing data product") # self.dpsc_cli.delete_data_product(dp_id) # Shut down container #container.stop() def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ log.debug('test_createDataProduct: Creating new data product w/o a stream definition (L4-CI-SA-RQ-308)') # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) log.debug('new dp_id = %s' % dp_id) log.debug("test_createDataProduct: Data product info from registry %s (L4-CI-SA-RQ-308)", str(dp_obj)) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) self.get_datastore(dataset_ids[0]) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) # pid = self.container.spawn_process(name='dummy_process_for_test', # module='pyon.ion.process', # cls='SimpleProcess', # config={}) # dummy_process = self.container.proc_manager.procs[pid] self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object try: dp_obj = self.rrclient.read(dp_id) except NotFound as ex: pass else: self.fail("force_deleted data product was found during read")
class TestObservatoryManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.org_management_service = OrgManagementServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #print 'TestObservatoryManagementServiceIntegration: started services' self.event_publisher = EventPublisher() # @unittest.skip('this exists only for debugging the launch process') # def test_just_the_setup(self): # return def destroy(self, resource_ids): self.OMS.force_delete_observatory(resource_ids.observatory_id) self.OMS.force_delete_subsite(resource_ids.subsite_id) self.OMS.force_delete_subsite(resource_ids.subsite2_id) self.OMS.force_delete_subsite(resource_ids.subsiteb_id) self.OMS.force_delete_subsite(resource_ids.subsitez_id) self.OMS.force_delete_platform_site(resource_ids.platform_site_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb2_id) self.OMS.force_delete_platform_site(resource_ids.platform_site3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site2_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_siteb3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site4_id) #@unittest.skip('targeting') def test_resources_associations(self): resources = self._make_associations() self.destroy(resources) #@unittest.skip('targeting') def test_find_related_frames_of_reference(self): # finding subordinates gives a dict of obj lists, convert objs to ids def idify(adict): ids = {} for k, v in adict.iteritems(): ids[k] = [] for obj in v: ids[k].append(obj._id) return ids # a short version of the function we're testing, with id-ify def short(resource_id, output_types): ret = self.OMS.find_related_frames_of_reference(resource_id, output_types) return idify(ret) #set up associations first stuff = self._make_associations() #basic traversal of tree from instrument to platform ids = short(stuff.instrument_site_id, [RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) #since this is the first search, just make sure the input inst_id got stripped if RT.InstrumentSite in ids: self.assertNotIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #basic traversal of tree from platform to instrument ids = short(stuff.platform_siteb_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) self.assertNotIn(stuff.instrument_site2_id, ids[RT.InstrumentSite]) #full traversal of tree from observatory down to instrument ids = short(stuff.observatory_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #full traversal of tree from instrument to observatory ids = short(stuff.instrument_site_id, [RT.Observatory]) self.assertIn(RT.Observatory, ids) self.assertIn(stuff.observatory_id, ids[RT.Observatory]) #partial traversal, only down to platform ids = short(stuff.observatory_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site3_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(RT.InstrumentSite, ids) #partial traversal, only down to platform ids = short(stuff.instrument_site_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertNotIn(RT.Observatory, ids) self.destroy(stuff) def _make_associations(self): """ create one of each resource and association used by OMS to guard against problems in ion-definitions """ #raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41") """ the tree we're creating (observatory, sites, platforms, instruments) rows are lettered, colums numbered. - first row is implied a - first column is implied 1 - site Z, just because O--Sz | S--S2--P3--I4 | Sb-Pb2-Ib3 | P--I2 <- PlatformDevice, InstrumentDevice2 | Pb <- PlatformDevice b | I <- InstrumentDevice """ org_id = self.OMS.create_marine_facility(any_old(RT.Org)) def create_under_org(resource_type): obj = any_old(resource_type) if RT.InstrumentDevice == resource_type: resource_id = self.IMS.create_instrument_device(obj) else: resource_id, _ = self.RR.create(obj) self.OMS.assign_resource_to_observatory_org(resource_id=resource_id, org_id=org_id) return resource_id #stuff we control observatory_id = create_under_org(RT.Observatory) subsite_id = create_under_org(RT.Subsite) subsite2_id = create_under_org(RT.Subsite) subsiteb_id = create_under_org(RT.Subsite) subsitez_id = create_under_org(RT.Subsite) platform_site_id = create_under_org(RT.PlatformSite) platform_siteb_id = create_under_org(RT.PlatformSite) platform_siteb2_id = create_under_org(RT.PlatformSite) platform_site3_id = create_under_org(RT.PlatformSite) instrument_site_id = create_under_org(RT.InstrumentSite) instrument_site2_id = create_under_org(RT.InstrumentSite) instrument_siteb3_id = create_under_org(RT.InstrumentSite) instrument_site4_id = create_under_org(RT.InstrumentSite) #stuff we associate to instrument_device_id = create_under_org(RT.InstrumentDevice) instrument_device2_id = create_under_org(RT.InstrumentDevice) platform_device_id = create_under_org(RT.PlatformDevice) platform_deviceb_id = create_under_org(RT.PlatformDevice) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) deployment_id, _ = self.RR.create(any_old(RT.Deployment)) #observatory self.RR.create_association(observatory_id, PRED.hasSite, subsite_id) self.RR.create_association(observatory_id, PRED.hasSite, subsitez_id) #site self.RR.create_association(subsite_id, PRED.hasSite, subsite2_id) self.RR.create_association(subsite_id, PRED.hasSite, subsiteb_id) self.RR.create_association(subsite2_id, PRED.hasSite, platform_site3_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_siteb2_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_site_id) #platform_site(s) self.RR.create_association(platform_site3_id, PRED.hasSite, instrument_site4_id) self.RR.create_association(platform_siteb2_id, PRED.hasSite, instrument_siteb3_id) self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site2_id) self.RR.create_association(platform_site_id, PRED.hasSite, platform_siteb_id) self.RR.create_association(platform_siteb_id, PRED.hasSite, instrument_site_id) self.RR.create_association(platform_siteb_id, PRED.hasDevice, platform_deviceb_id) self.RR.create_association(platform_site_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id) self.RR.create_association(platform_site_id, PRED.hasDeployment, deployment_id) #instrument_site(s) self.RR.create_association(instrument_site_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(instrument_site_id, PRED.hasDeployment, deployment_id) self.RR.create_association(instrument_site2_id, PRED.hasDevice, instrument_device2_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id) ret = DotDict() ret.org_id = org_id ret.observatory_id = observatory_id ret.subsite_id = subsite_id ret.subsite2_id = subsite2_id ret.subsiteb_id = subsiteb_id ret.subsitez_id = subsitez_id ret.platform_site_id = platform_site_id ret.platform_siteb_id = platform_siteb_id ret.platform_siteb2_id = platform_siteb2_id ret.platform_site3_id = platform_site3_id ret.instrument_site_id = instrument_site_id ret.instrument_site2_id = instrument_site2_id ret.instrument_siteb3_id = instrument_siteb3_id ret.instrument_site4_id = instrument_site4_id ret.instrument_device_id = instrument_device_id ret.instrument_device2_id = instrument_device2_id ret.platform_device_id = platform_device_id ret.platform_deviceb_id = platform_deviceb_id ret.instrument_model_id = instrument_model_id ret.platform_model_id = platform_model_id ret.deployment_id = deployment_id return ret #@unittest.skip("targeting") def test_create_observatory(self): observatory_obj = IonObject(RT.Observatory, name='TestFacility', description='some new mf') observatory_id = self.OMS.create_observatory(observatory_obj) self.OMS.force_delete_observatory(observatory_id) #@unittest.skip("targeting") def test_find_observatory_org(self): org_obj = IonObject(RT.Org, name='TestOrg', description='some new mf org') org_id = self.OMS.create_marine_facility(org_obj) observatory_obj = IonObject(RT.Observatory, name='TestObservatory', description='some new obs') observatory_id = self.OMS.create_observatory(observatory_obj) #make association self.OMS.assign_resource_to_observatory_org(observatory_id, org_id) #find association org_objs = self.OMS.find_org_by_observatory(observatory_id) self.assertEqual(1, len(org_objs)) self.assertEqual(org_id, org_objs[0]._id) print("org_id=<" + org_id + ">") #create a subsite with parent Observatory subsite_obj = IonObject(RT.Subsite, name= 'TestSubsite', description = 'sample subsite') subsite_id = self.OMS.create_subsite(subsite_obj, observatory_id) self.assertIsNotNone(subsite_id, "Subsite not created.") # verify that Subsite is linked to Observatory mf_subsite_assoc = self.RR.get_association(observatory_id, PRED.hasSite, subsite_id) self.assertIsNotNone(mf_subsite_assoc, "Subsite not connected to Observatory.") # add the Subsite as a resource of this Observatory self.OMS.assign_resource_to_observatory_org(resource_id=subsite_id, org_id=org_id) # verify that Subsite is linked to Org org_subsite_assoc = self.RR.get_association(org_id, PRED.hasResource, subsite_id) self.assertIsNotNone(org_subsite_assoc, "Subsite not connected as resource to Org.") #create a logical platform with parent Subsite platform_site_obj = IonObject(RT.PlatformSite, name= 'TestPlatformSite', description = 'sample logical platform') platform_site_id = self.OMS.create_platform_site(platform_site_obj, subsite_id) self.assertIsNotNone(platform_site_id, "PlatformSite not created.") # verify that PlatformSite is linked to Site site_lp_assoc = self.RR.get_association(subsite_id, PRED.hasSite, platform_site_id) self.assertIsNotNone(site_lp_assoc, "PlatformSite not connected to Site.") # add the PlatformSite as a resource of this Observatory self.OMS.assign_resource_to_observatory_org(resource_id=platform_site_id, org_id=org_id) # verify that PlatformSite is linked to Org org_lp_assoc = self.RR.get_association(org_id, PRED.hasResource, platform_site_id) self.assertIsNotNone(org_lp_assoc, "PlatformSite not connected as resource to Org.") #create a logical instrument with parent logical platform instrument_site_obj = IonObject(RT.InstrumentSite, name= 'TestInstrumentSite', description = 'sample logical instrument') instrument_site_id = self.OMS.create_instrument_site(instrument_site_obj, platform_site_id) self.assertIsNotNone(instrument_site_id, "InstrumentSite not created.") # verify that InstrumentSite is linked to PlatformSite li_lp_assoc = self.RR.get_association(platform_site_id, PRED.hasSite, instrument_site_id) self.assertIsNotNone(li_lp_assoc, "InstrumentSite not connected to PlatformSite.") # add the InstrumentSite as a resource of this Observatory self.OMS.assign_resource_to_observatory_org(resource_id=instrument_site_id, org_id=org_id) # verify that InstrumentSite is linked to Org org_li_assoc = self.RR.get_association(org_id, PRED.hasResource, instrument_site_id) self.assertIsNotNone(org_li_assoc, "InstrumentSite not connected as resource to Org.") # remove the InstrumentSite as a resource of this Observatory self.OMS.unassign_resource_from_observatory_org(instrument_site_id, org_id) # verify that InstrumentSite is linked to Org assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.InstrumentSite, id_only=True ) self.assertEqual(len(assocs), 0) # remove the InstrumentSite self.OMS.delete_instrument_site(instrument_site_id) assocs, _ = self.RR.find_objects(platform_site_id, PRED.hasSite, RT.InstrumentSite, id_only=True ) self.assertEqual(len(assocs), 1) #todo: remove the dangling association # remove the PlatformSite as a resource of this Observatory self.OMS.unassign_resource_from_observatory_org(platform_site_id, org_id) # verify that PlatformSite is linked to Org assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.PlatformSite, id_only=True ) self.assertEqual(len(assocs), 0) # remove the Site as a resource of this Observatory self.OMS.unassign_resource_from_observatory_org(subsite_id, org_id) # verify that Site is linked to Org assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.Subsite, id_only=True ) self.assertEqual(len(assocs), 0) self.RR.delete(org_id) self.OMS.force_delete_observatory(observatory_id) self.OMS.force_delete_subsite(subsite_id) self.OMS.force_delete_platform_site(platform_site_id) self.OMS.force_delete_instrument_site(instrument_site_id) #@unittest.skip("in development...") @attr('EXT') def test_observatory_org_extended(self): stuff = self._make_associations() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.damsclient.assign_data_product(input_resource_id=stuff.instrument_device_id, data_product_id=data_product_id1) #-------------------------------------------------------------------------------- # Get the extended Site (platformSite) #-------------------------------------------------------------------------------- extended_site = self.OMS.get_site_extension(stuff.platform_site_id) log.debug("extended_site: %s ", str(extended_site)) self.assertEqual(1, len(extended_site.platform_devices)) self.assertEqual(1, len(extended_site.platform_models)) self.assertEqual(stuff.platform_device_id, extended_site.platform_devices[0]._id) self.assertEqual(stuff.platform_model_id, extended_site.platform_models[0]._id) #-------------------------------------------------------------------------------- # Get the extended Org #-------------------------------------------------------------------------------- #test the extended resource extended_org = self.org_management_service.get_marine_facility_extension(stuff.org_id) log.debug("test_observatory_org_extended: extended_org: %s ", str(extended_org)) #self.assertEqual(2, len(extended_org.instruments_deployed) ) #self.assertEqual(1, len(extended_org.platforms_not_deployed) ) self.assertEqual(2, extended_org.number_of_platforms) self.assertEqual(2, len(extended_org.platform_models) ) self.assertEqual(2, extended_org.number_of_instruments) self.assertEqual(2, len(extended_org.instrument_models) ) #test the extended resource of the ION org ion_org_id = self.org_management_service.find_org() extended_org = self.org_management_service.get_marine_facility_extension(ion_org_id._id) log.debug("test_observatory_org_extended: extended_ION_org: %s ", str(extended_org)) self.assertEqual(0, len(extended_org.members)) self.assertEqual(0, extended_org.number_of_platforms) #self.assertEqual(1, len(extended_org.sites)) #-------------------------------------------------------------------------------- # Get the extended Site #-------------------------------------------------------------------------------- #create device state events to use for op /non-op filtering in extended t = get_ion_ts() self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent', origin = stuff.instrument_device_id, state=ResourceAgentState.STREAMING ) self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent', origin = stuff.instrument_device2_id, state=ResourceAgentState.INACTIVE ) extended_site = self.OMS.get_site_extension(stuff.instrument_site2_id) log.debug("test_observatory_org_extended: extended_site: %s ", str(extended_site)) self.dpclient.delete_data_product(data_product_id1)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient() self.rrclient = ResourceRegistryServiceClient() self.damsclient = DataAcquisitionManagementServiceClient() self.pubsubcli = PubsubManagementServiceClient() self.ingestclient = IngestionManagementServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ datastore_name = CACHE_DATASTORE_NAME self.db = self.container.datastore_manager.get_datastore(datastore_name) self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.process_dispatcher.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) IngestionManagementIntTest.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore @attr('EXT') @attr('PREP') def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Generic time-series data domain creation tdom, sdom = time_series_domain() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0 dp_obj.ooi_product_name = "PRODNAME" #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id) # Assert that the data product has an associated stream at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertNotEquals(len(stream_ids), 0) # Assert that the data product has an associated stream def at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True) self.assertNotEquals(len(stream_ids), 0) self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Created data product %s', dp_obj) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id2) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) group_names = self.dpsc_cli.get_data_product_group_list() self.assertIn("PRODNAME", group_names) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0 # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description,'the very first dp') self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Updated data product %s', dp_obj) #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) self.assertEqual(dp_id, extended_product._id) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual(0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) def ion_object_encoder(obj): return obj.__dict__ #test prepare for create data_product_data = self.dpsc_cli.prepare_data_product_support() #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, "") self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0) self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 0) self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0) #test prepare for update data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id) #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, dp_id) self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual(len(data_product_data.associations['Dataset'].resources), 1) self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1) self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, dp_id) self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 1) self.assertEqual(data_product_data.associations['Dataset'].associated_resources[0].s, dp_id) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) # Assert that there are no associated streams leftover after deleting the data product stream_ids, assoc_ids = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertEquals(len(stream_ids), 0) self.assertEquals(len(assoc_ids), 0) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product(dp_id) # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) self.assertTrue(len(events) > 0) def test_data_product_stream_def(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id) self.assertEquals(ctd_stream_def_id, stream_def_id) def test_derived_data_product(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='ctd parsed', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id) tdom, sdom = time_series_domain() dp = DataProduct(name='Instrument DP', temporal_domain=tdom.dump(), spatial_domain=sdom.dump()) dp_id = self.dpsc_cli.create_data_product(dp, stream_definition_id=ctd_stream_def_id) self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id) self.dpsc_cli.activate_data_product_persistence(dp_id) self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Make the derived data product simple_stream_def_id = self.pubsubcli.create_stream_definition(name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time','temp']) tempwat_dp = DataProduct(name='TEMPWAT') tempwat_dp_id = self.dpsc_cli.create_data_product(tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id) self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id) # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) rdt['pressure'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id,route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True) tempwat_dataset_id = tempwat_dataset_ids[0] granule = self.data_retriever.retrieve(tempwat_dataset_id, delivery_format=simple_stream_def_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['time'], np.arange(20)) self.assertEquals(set(rdt.fields), set(['time','temp'])) def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id,route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_ids[0]) self.assertIsInstance(replay_data, Granule) log.debug("The data retriever was able to replay the dataset that was attached to the data product " "we wanted to be persisted. Therefore the data product was indeed persisted with " "otherwise we could not have retrieved its dataset using the data retriever. Therefore " "this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'") data_product_object = self.rrclient.read(dp_id) self.assertEquals(data_product_object.name,'DP1') self.assertEquals(data_product_object.description,'some new dp') log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. " " Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the " "resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name, data_product_object.description)) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) dataset_modified.clear() rdt['time'] = np.arange(20,40) publisher.publish(rdt.to_granule()) self.assertFalse(dataset_modified.wait(2)) self.dpsc_cli.activate_data_product_persistence(dp_id) dataset_modified.clear() publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_almost_equal(rdt['time'], np.arange(40)) dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True) self.assertEquals(len(dataset_ids), 1) self.dpsc_cli.suspend_data_product_persistence(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.rrclient.read(dp_id) def test_lookup_values(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_lookups() stream_def_id = self.pubsubcli.create_stream_definition('lookup', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsubcli.delete_stream_definition, stream_def_id) data_product = DataProduct(name='lookup data product') tdom, sdom = time_series_domain() data_product.temporal_domain = tdom.dump() data_product.spatial_domain = sdom.dump() data_product_id = self.dpsc_cli.create_data_product(data_product, stream_definition_id=stream_def_id) self.addCleanup(self.dpsc_cli.delete_data_product, data_product_id) data_producer = DataProducer(name='producer') data_producer.producer_context = DataProcessProducerContext() data_producer.producer_context.configuration['qc_keys'] = ['offset_document'] data_producer_id, _ = self.rrclient.create(data_producer) self.addCleanup(self.rrclient.delete, data_producer_id) assoc,_ = self.rrclient.create_association(subject=data_product_id, object=data_producer_id, predicate=PRED.hasDataProducer) self.addCleanup(self.rrclient.delete_association, assoc) document_keys = self.damsclient.list_qc_references(data_product_id) self.assertEquals(document_keys, ['offset_document']) svm = StoredValueManager(self.container) svm.stored_value_cas('offset_document', {'offset_a':2.0}) self.dpsc_cli.activate_data_product_persistence(data_product_id) dataset_ids, _ = self.rrclient.find_objects(subject=data_product_id, predicate=PRED.hasDataset, id_only=True) dataset_id = dataset_ids[0] dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = [0] rdt['temp'] = [20.] granule = rdt.to_granule() stream_ids, _ = self.rrclient.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) publisher = StandaloneStreamPublisher(stream_id, route) publisher.publish(granule) self.assertTrue(dataset_monitor.event.wait(10)) granule = self.data_retriever.retrieve(dataset_id) rdt2 = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['temp'], rdt2['temp']) np.testing.assert_array_almost_equal(rdt2['calibrated'], np.array([22.0])) svm.stored_value_cas('updated_document', {'offset_a':3.0}) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) ep = EventPublisher(event_type=OT.ExternalReferencesUpdatedEvent) ep.publish_event(origin=data_product_id, reference_keys=['updated_document']) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = [1] rdt['temp'] = [20.] granule = rdt.to_granule() gevent.sleep(2) # Yield so that the event goes through publisher.publish(granule) self.assertTrue(dataset_monitor.event.wait(10)) granule = self.data_retriever.retrieve(dataset_id) rdt2 = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt2['temp'],np.array([20.,20.])) np.testing.assert_array_almost_equal(rdt2['calibrated'], np.array([22.0,23.0]))
class TestDataProductVersions(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.client = DataProductManagementServiceClient( node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #@unittest.skip('not working') @patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}}) def test_createDataProductVersionSimple(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='test', parameter_dictionary_id=pdict_id) # test creating a new data product which will also create the initial/default version log.debug('Creating new data product with a stream definition') # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP', description='some new dp', temporal_domain=tdom, spatial_domain=sdom) dp_id = self.client.create_data_product(dp_obj, ctd_stream_def_id) log.debug('new dp_id = %s', str(dp_id)) dpc_id = self.client.create_data_product_collection( data_product_id=dp_id, collection_name='firstCollection', collection_description='collection desc') #test that the links exist version_ids, _ = self.rrclient.find_objects(subject=dpc_id, predicate=PRED.hasVersion, id_only=True) log.debug('version_ids = %s', str(version_ids)) self.assertTrue( version_ids, 'Failed to connect the data product to the version collection.') self.assertTrue( version_ids[0] == dp_id, 'Failed to connect the data product to the version collection.') # test creating a subsequent data product version which will update the data product pointers dp2_obj = IonObject(RT.DataProduct, name='DP2', description='a second dp', temporal_domain=tdom, spatial_domain=sdom) dp2_id = self.client.create_data_product(dp2_obj, ctd_stream_def_id) log.debug('second dp_id = %s', dp2_id) log.debug("adding product %s to collection %s", dp2_id, dpc_id) self.client.add_data_product_version_to_collection( data_product_id=dp2_id, data_product_collection_id=dpc_id, version_name="second version", version_description="a second version created") #test that the links exist version_ids = self.RR2.find_data_product_ids_of_data_product_collection_using_has_version( dpc_id) self.assertEqual(2, len(version_ids)) recent_version_id = self.client.get_current_version(dpc_id) self.assertEquals(recent_version_id, dp2_id) base_version_id = self.client.get_base_version(dpc_id) self.assertEquals(base_version_id, dp_id) #--------------------------------------------------------------------------------------------- # Now check that we can subscribe to the stream for the data product version #--------------------------------------------------------------------------------------------- # Activating the data products contained in the versions held by the data product collection data_product_collection_obj = self.rrclient.read(dpc_id) version_list = data_product_collection_obj.version_list self.assertEquals(len(version_list), 2) for version in version_list: data_product_id = version.data_product_id self.client.activate_data_product_persistence(data_product_id) streams = self.RR2.find_streams_of_data_product_using_has_stream( data_product_id) self.assertNotEqual(0, len(streams)) for stream in streams: self.assertTrue(stream.persisted) log.debug( "This satisfies L4-CI-DM-RQ-053: 'The dynamic data distribution services shall support multiple " "versions of a given data topic.' This is true because we have shown above that we can persist the " "streams associated to the data product versions, and therefore we can subscribe to them. In " "test_oms_launch2.py, we have tested that activated data products like the ones we have here can be " "used by data processes to gather streaming data and it all works together correctly. This therefore " "completes the demonstration of req L4-CI-DM-RQ-053.") #--------------------------------------------------------------------------------------------- # Now delete all the created stuff and look for possible problems in doing so #--------------------------------------------------------------------------------------------- log.debug("deleting all data productgs") self.client.delete_data_product(dp_id) self.client.delete_data_product(dp2_id) self.client.delete_data_product_collection(dpc_id) self.client.force_delete_data_product_collection(dpc_id) # now try to get the deleted dp object self.assertRaises(NotFound, self.client.read_data_product_collection, dpc_id)
class TestGranulePublish(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid #overriding trigger function here to use new granule def test_granule_publish(self): log.debug("test_granule_publish ") self.loggerpids = [] #retrieve the param dict from the repository pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) stream_definition_id = self.pubsubclient.create_stream_definition( 'parsed stream', parameter_dictionary_id=pdict_id) tdom, sdom = time_series_domain() dp_obj = IonObject(RT.DataProduct, name=str(uuid.uuid4()), description='ctd stream test', temporal_domain=tdom.dump(), spatial_domain=sdom.dump()) data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=stream_definition_id) # Retrieve the id of the output stream of the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('test_granule_publish: Data product streams1 = %s', stream_ids) pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id) #create the publisher from the stream route stream_route = self.pubsubclient.read_stream_route(stream_ids[0]) publisher = StandaloneStreamPublisher(stream_ids[0], stream_route) # this is one sample from the ctd driver tomato = { "driver_timestamp": 3555971105.1268806, "instrument_id": "ABC-123", "pkt_format_id": "JSON_Data", "pkt_version": 1, "preferred_timestamp": "driver_timestamp", "quality_flag": "ok", "stream_name": "parsed", "values": [{ "value": 22.9304, "value_id": "temp" }, { "value": 51.57381, "value_id": "conductivity" }, { "value": 915.551, "value_id": "pressure" }] } for value in tomato['values']: log.debug( "test_granule_publish: Looping tomato values key: %s val: %s ", str(value['value']), str(value['value_id'])) if value['value_id'] in rdt: rdt[value['value_id']] = numpy.array([value['value']]) log.debug( "test_granule_publish: Added data item %s val: %s ", str(value['value']), str(value['value_id'])) g = rdt.to_granule() publisher.publish(g) gevent.sleep(3) for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) #-------------------------------------------------------------------------------- # Cleanup data products #-------------------------------------------------------------------------------- dp_ids, _ = self.rrclient.find_resources(restype=RT.DataProduct, id_only=True) for dp_id in dp_ids: self.dataproductclient.delete_data_product(dp_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) print 'started services' # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain = tdom, spatial_domain = sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent[0].name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension(platform_device_id) self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(StatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR resource_impl = ResourceImpl(c) resource_impl.pluck(instrument_agent_id) resource_impl.pluck(instrument_model_id) resource_impl.pluck(instrument_device_id) resource_impl.pluck(platform_agent_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": {"favorite_color": "attr desc goes here"} })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) print "Saved config:" print snap_obj.content #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2)
class TestAgentLaunchOps(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' unittest # suppress an pycharm inspector error if all unittest.skip references are commented out self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient( node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return def test_get_agent_client_noprocess(self): inst_device_id = self.RR2.create(any_old(RT.InstrumentDevice)) iap = ResourceAgentClient._get_agent_process_id(inst_device_id) # should be no running agent self.assertIsNone(iap) # should raise NotFound self.assertRaises(NotFound, ResourceAgentClient, inst_device_id) def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.IMS.create_instrument_device( instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) spdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition( name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition( name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test') data_product_id1 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence( data_product_id=data_product_id1) self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test') data_product_id2 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence( data_product_id=data_product_id2) self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance( instAgentInstance_id) gate = AgentProcessStateGate(self.PDC.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) if "BAD_DATA" == instance_obj.driver_config["comms_config"]: print "Saved config:" print snap_obj.content self.fail("Saved config was not properly restored") self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config_hasDevice(self): def assign_fn(child_device_id, parent_device_id): self.RR2.create_association(parent_device_id, PRED.hasDevice, child_device_id) def find_fn(parent_device_id): ret, _ = self.RR.find_objects(subject=parent_device_id, predicate=PRED.hasDevice, id_only=True) return ret self.base_agent_instance_config(assign_fn, find_fn) log.info("END test_agent_instance_config_hasDevice") def test_agent_instance_config_hasNetworkParent(self): def assign_fn(child_device_id, parent_device_id): self.RR2.create_association(child_device_id, PRED.hasNetworkParent, parent_device_id) def find_fn(parent_device_id): ret, _ = self.RR.find_subjects(object=parent_device_id, predicate=PRED.hasNetworkParent, id_only=True) return ret self.base_agent_instance_config(assign_fn, find_fn) log.info("END test_agent_instance_config_hasNetworkParent") def base_agent_instance_config( self, assign_child_platform_to_parent_platform_fn, find_child_platform_ids_of_parent_platform_fn): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC config_builder = DotDict config_builder.i = None config_builder.p = None def refresh_pconfig_builder_hack(config_builder): """ ugly hack to get around "idempotent" RR2 caching remove after https://github.com/ooici/coi-services/pull/1190 """ config_builder.p = PlatformAgentConfigurationBuilder(clients) def refresh_iconfig_builder_hack(config_builder): """ ugly hack to get around "idempotent" RR2 caching remove after https://github.com/ooici/coi-services/pull/1190 """ config_builder.i = InstrumentAgentConfigurationBuilder(clients) org_obj = any_old(RT.Org) org_id = self.RR2.create(org_obj) inst_startup_config = {'startup': 'config'} generic_alerts_config = [{'lvl2': 'lvl3val'}] required_config_keys = [ 'org_governance_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'aparam_alerts_config', 'children' ] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = { 'process_type': ('ZMQPyClassDriverLauncher', ), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertIn('resource_id', config['agent']) self.assertEqual(device_id, config['agent']['resource_id']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) for key in ['children']: self.assertEqual({}, config[key]) # TODO(OOIION-1495) review the asserts below related with # requiring 'ports' to be present in the driver_config. # See recent adjustment in agent_configuration_builder.py, # which I did to avoid other tests to fail. # The asserts below would make the following tests fail: # test_agent_instance_config_hasDevice # test_agent_instance_config_hasNetworkParent def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertIn('resource_id', config['agent']) self.assertEqual(device_id, config['agent']['resource_id']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) self.assertIn('driver_config', config) self.assertIn('foo', config['driver_config']) """ self.assertIn('ports', config['driver_config']) """ self.assertEqual('bar', config['driver_config']['foo']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher', ), config['driver_config']['process_type']) if None is inst_device_id: for key in ['children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) """ if config['driver_config']['ports']: self.assertTrue( isinstance(config['driver_config']['ports'], dict) ) """ def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertIn('process_type', config['driver_config']) """ self.assertIn('ports', config['driver_config']) """ self.assertEqual(('ZMQPyClassDriverLauncher', ), config['driver_config']['process_type']) self.assertIn('resource_id', config['agent']) self.assertEqual(parent_device_id, config['agent']['resource_id']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) for key in ['startup_config']: self.assertEqual({}, config[key]) """ if config['driver_config']['ports']: self.assertTrue( isinstance(config['driver_config']['ports'], dict) ) """ self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition( name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(name='', agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old( RT.PlatformAgentInstance, { 'driver_config': { 'foo': 'bar' }, 'alerts': generic_alerts_config }) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance( platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') platform_agent_obj = any_old( RT.PlatformAgent, {"stream_configurations": [raw_config]}) platform_agent_id = self.IMS.create_platform_agent( platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device( any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct) dp_id = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.suspend_data_product_persistence, dp_id) #deployment creation site_obj = IonObject(RT.PlatformSite, name='sitePlatform') site_id = self.OMS.create_platform_site(platform_site=site_obj) # find current deployment using time constraints current_time = int(calendar.timegm(time.gmtime())) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) platform_port_obj = IonObject( OT.PlatformPort, reference_designator='GA01SUMO-FI003-09-CTDMO0999', port_type=PortTypeEnum.UPLINK, ip_address=0) deployment_obj = IonObject( RT.Deployment, name='TestPlatformDeployment_' + name, description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds], port_assignments={platform_device_id: platform_port_obj}) deploy_id = self.OMS.create_deployment( deployment=deployment_obj, site_id=site_id, device_id=platform_device_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance( platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition( platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource( platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old( RT.InstrumentAgentInstance, { "startup_config": inst_startup_config, 'alerts': generic_alerts_config }) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance( instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') instrument_agent_obj = any_old( RT.InstrumentAgent, {"stream_configurations": [raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent( instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device( any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct) dp_id = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product( input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.suspend_data_product_persistence, dp_id) #deployment creation site_obj = IonObject(RT.InstrumentSite, name='siteInstrument') site_id = self.OMS.create_instrument_site(instrument_site=site_obj) # find current deployment using time constraints current_time = int(calendar.timegm(time.gmtime())) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) platform_port_obj = IonObject( OT.PlatformPort, reference_designator='GA01SUMO-FI003-08-CTDMO0888', port_type=PortTypeEnum.PAYLOAD, ip_address=0) deployment_obj = IonObject( RT.Deployment, name='TestDeployment for Cabled Instrument', description='some new deployment', context=IonObject(OT.CabledInstrumentDeploymentContext), constraint_list=[temporal_bounds], port_assignments={instrument_device_id: platform_port_obj}) deploy_id = self.OMS.create_deployment( deployment=deployment_obj, site_id=site_id, device_id=instrument_device_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device_with_has_agent_instance( instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance_with_has_agent_definition( instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource( instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug( "Testing that preparing a launcher without agent instance raises an error" ) refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated self.assertRaises(AssertionError, config_builder.p.prepare, will_launch=False) log.debug( "Making the structure for a platform agent, which will be the child" ) platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure( name='child') platform_agent_instance_child_obj = self.RR2.read( platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object( platform_agent_instance_child_obj) child_config = config_builder.p.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug( "Making the structure for a platform agent, which will be the parent" ) platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure( name='parent') platform_agent_instance_parent_obj = self.RR2.read( platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object( platform_agent_instance_parent_obj) parent_config = config_builder.p.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") assign_child_platform_to_parent_platform_fn(platform_device_child_id, platform_device_parent_id) child_device_ids = find_child_platform_ids_of_parent_platform_fn( platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object( platform_agent_instance_parent_obj) parent_config = config_builder.p.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure( ) instrument_agent_instance_obj = self.RR2.read( instrument_agent_instance_id) log.debug("Testing instrument config") refresh_iconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.i.set_agent_instance_object( instrument_agent_instance_obj) instrument_config = config_builder.i.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device_with_has_device( instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_platform_device_using_has_device( platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object( platform_agent_instance_parent_obj) full_config = config_builder.p.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id) #self.fail(parent_config) #plauncher.prepare(will_launch=False) log.info("END base_agent_instance_config")
class TestObservatoryManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.org_management_service = OrgManagementServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #print 'TestObservatoryManagementServiceIntegration: started services' self.event_publisher = EventPublisher() # @unittest.skip('this exists only for debugging the launch process') # def test_just_the_setup(self): # return def destroy(self, resource_ids): self.OMS.force_delete_observatory(resource_ids.observatory_id) self.OMS.force_delete_subsite(resource_ids.subsite_id) self.OMS.force_delete_subsite(resource_ids.subsite2_id) self.OMS.force_delete_subsite(resource_ids.subsiteb_id) self.OMS.force_delete_subsite(resource_ids.subsitez_id) self.OMS.force_delete_platform_site(resource_ids.platform_site_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb2_id) self.OMS.force_delete_platform_site(resource_ids.platform_site3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site2_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_siteb3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site4_id) #@unittest.skip('targeting') def test_resources_associations(self): resources = self._make_associations() self.destroy(resources) #@unittest.skip('targeting') def test_find_related_frames_of_reference(self): # finding subordinates gives a dict of obj lists, convert objs to ids def idify(adict): ids = {} for k, v in adict.iteritems(): ids[k] = [] for obj in v: ids[k].append(obj._id) return ids # a short version of the function we're testing, with id-ify def short(resource_id, output_types): ret = self.OMS.find_related_frames_of_reference(resource_id, output_types) return idify(ret) #set up associations first stuff = self._make_associations() #basic traversal of tree from instrument to platform ids = short(stuff.instrument_site_id, [RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) #since this is the first search, just make sure the input inst_id got stripped if RT.InstrumentSite in ids: self.assertNotIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #basic traversal of tree from platform to instrument ids = short(stuff.platform_siteb_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) self.assertNotIn(stuff.instrument_site2_id, ids[RT.InstrumentSite]) #full traversal of tree from observatory down to instrument ids = short(stuff.observatory_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #full traversal of tree from instrument to observatory ids = short(stuff.instrument_site_id, [RT.Observatory]) self.assertIn(RT.Observatory, ids) self.assertIn(stuff.observatory_id, ids[RT.Observatory]) #partial traversal, only down to platform ids = short(stuff.observatory_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site3_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(RT.InstrumentSite, ids) #partial traversal, only down to platform ids = short(stuff.instrument_site_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertNotIn(RT.Observatory, ids) self.destroy(stuff) def _make_associations(self): """ create one of each resource and association used by OMS to guard against problems in ion-definitions """ #raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41") """ the tree we're creating (observatory, sites, platforms, instruments) rows are lettered, colums numbered. - first row is implied a - first column is implied 1 - site Z, just because O--Sz | S--S2--P3--I4 | Sb-Pb2-Ib3 | P--I2 <- PlatformDevice, InstrumentDevice2 | Pb <- PlatformDevice b | I <- InstrumentDevice """ org_id = self.OMS.create_marine_facility(any_old(RT.Org)) def create_under_org(resource_type): obj = any_old(resource_type) if RT.InstrumentDevice == resource_type: resource_id = self.IMS.create_instrument_device(obj) else: resource_id, _ = self.RR.create(obj) self.OMS.assign_resource_to_observatory_org(resource_id=resource_id, org_id=org_id) return resource_id #stuff we control observatory_id = create_under_org(RT.Observatory) subsite_id = create_under_org(RT.Subsite) subsite2_id = create_under_org(RT.Subsite) subsiteb_id = create_under_org(RT.Subsite) subsitez_id = create_under_org(RT.Subsite) platform_site_id = create_under_org(RT.PlatformSite) platform_siteb_id = create_under_org(RT.PlatformSite) platform_siteb2_id = create_under_org(RT.PlatformSite) platform_site3_id = create_under_org(RT.PlatformSite) instrument_site_id = create_under_org(RT.InstrumentSite) instrument_site2_id = create_under_org(RT.InstrumentSite) instrument_siteb3_id = create_under_org(RT.InstrumentSite) instrument_site4_id = create_under_org(RT.InstrumentSite) #stuff we associate to instrument_device_id = create_under_org(RT.InstrumentDevice) instrument_device2_id = create_under_org(RT.InstrumentDevice) platform_device_id = create_under_org(RT.PlatformDevice) platform_deviceb_id = create_under_org(RT.PlatformDevice) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) deployment_id, _ = self.RR.create(any_old(RT.Deployment)) #observatory self.RR.create_association(observatory_id, PRED.hasSite, subsite_id) self.RR.create_association(observatory_id, PRED.hasSite, subsitez_id) #site self.RR.create_association(subsite_id, PRED.hasSite, subsite2_id) self.RR.create_association(subsite_id, PRED.hasSite, subsiteb_id) self.RR.create_association(subsite2_id, PRED.hasSite, platform_site3_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_siteb2_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_site_id) #platform_site(s) self.RR.create_association(platform_site3_id, PRED.hasSite, instrument_site4_id) self.RR.create_association(platform_siteb2_id, PRED.hasSite, instrument_siteb3_id) self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site2_id) self.RR.create_association(platform_site_id, PRED.hasSite, platform_siteb_id) self.RR.create_association(platform_siteb_id, PRED.hasSite, instrument_site_id) self.RR.create_association(platform_siteb_id, PRED.hasDevice, platform_deviceb_id) self.RR.create_association(platform_site_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id) self.RR.create_association(platform_site_id, PRED.hasDeployment, deployment_id) #instrument_site(s) self.RR.create_association(instrument_site_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(instrument_site_id, PRED.hasDeployment, deployment_id) self.RR.create_association(instrument_site2_id, PRED.hasDevice, instrument_device2_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id) ret = DotDict() ret.org_id = org_id ret.observatory_id = observatory_id ret.subsite_id = subsite_id ret.subsite2_id = subsite2_id ret.subsiteb_id = subsiteb_id ret.subsitez_id = subsitez_id ret.platform_site_id = platform_site_id ret.platform_siteb_id = platform_siteb_id ret.platform_siteb2_id = platform_siteb2_id ret.platform_site3_id = platform_site3_id ret.instrument_site_id = instrument_site_id ret.instrument_site2_id = instrument_site2_id ret.instrument_siteb3_id = instrument_siteb3_id ret.instrument_site4_id = instrument_site4_id ret.instrument_device_id = instrument_device_id ret.instrument_device2_id = instrument_device2_id ret.platform_device_id = platform_device_id ret.platform_deviceb_id = platform_deviceb_id ret.instrument_model_id = instrument_model_id ret.platform_model_id = platform_model_id ret.deployment_id = deployment_id return ret #@unittest.skip("targeting") def test_create_observatory(self): observatory_obj = IonObject(RT.Observatory, name='TestFacility', description='some new mf') observatory_id = self.OMS.create_observatory(observatory_obj) self.OMS.force_delete_observatory(observatory_id) #@unittest.skip("targeting") def test_find_observatory_org(self): org_obj = IonObject(RT.Org, name='TestOrg', description='some new mf org') org_id = self.OMS.create_marine_facility(org_obj) observatory_obj = IonObject(RT.Observatory, name='TestObservatory', description='some new obs') observatory_id = self.OMS.create_observatory(observatory_obj) #make association self.OMS.assign_resource_to_observatory_org(observatory_id, org_id) #find association org_objs = self.OMS.find_org_by_observatory(observatory_id) self.assertEqual(1, len(org_objs)) self.assertEqual(org_id, org_objs[0]._id) print("org_id=<" + org_id + ">") #create a subsite with parent Observatory subsite_obj = IonObject(RT.Subsite, name= 'TestSubsite', description = 'sample subsite') subsite_id = self.OMS.create_subsite(subsite_obj, observatory_id) self.assertIsNotNone(subsite_id, "Subsite not created.") # verify that Subsite is linked to Observatory mf_subsite_assoc = self.RR.get_association(observatory_id, PRED.hasSite, subsite_id) self.assertIsNotNone(mf_subsite_assoc, "Subsite not connected to Observatory.") # add the Subsite as a resource of this Observatory self.OMS.assign_resource_to_observatory_org(resource_id=subsite_id, org_id=org_id) # verify that Subsite is linked to Org org_subsite_assoc = self.RR.get_association(org_id, PRED.hasResource, subsite_id) self.assertIsNotNone(org_subsite_assoc, "Subsite not connected as resource to Org.") #create a logical platform with parent Subsite platform_site_obj = IonObject(RT.PlatformSite, name= 'TestPlatformSite', description = 'sample logical platform') platform_site_id = self.OMS.create_platform_site(platform_site_obj, subsite_id) self.assertIsNotNone(platform_site_id, "PlatformSite not created.") # verify that PlatformSite is linked to Site site_lp_assoc = self.RR.get_association(subsite_id, PRED.hasSite, platform_site_id) self.assertIsNotNone(site_lp_assoc, "PlatformSite not connected to Site.") # add the PlatformSite as a resource of this Observatory self.OMS.assign_resource_to_observatory_org(resource_id=platform_site_id, org_id=org_id) # verify that PlatformSite is linked to Org org_lp_assoc = self.RR.get_association(org_id, PRED.hasResource, platform_site_id) self.assertIsNotNone(org_lp_assoc, "PlatformSite not connected as resource to Org.") #create a logical instrument with parent logical platform instrument_site_obj = IonObject(RT.InstrumentSite, name= 'TestInstrumentSite', description = 'sample logical instrument') instrument_site_id = self.OMS.create_instrument_site(instrument_site_obj, platform_site_id) self.assertIsNotNone(instrument_site_id, "InstrumentSite not created.") # verify that InstrumentSite is linked to PlatformSite li_lp_assoc = self.RR.get_association(platform_site_id, PRED.hasSite, instrument_site_id) self.assertIsNotNone(li_lp_assoc, "InstrumentSite not connected to PlatformSite.") # add the InstrumentSite as a resource of this Observatory self.OMS.assign_resource_to_observatory_org(resource_id=instrument_site_id, org_id=org_id) # verify that InstrumentSite is linked to Org org_li_assoc = self.RR.get_association(org_id, PRED.hasResource, instrument_site_id) self.assertIsNotNone(org_li_assoc, "InstrumentSite not connected as resource to Org.") # remove the InstrumentSite as a resource of this Observatory self.OMS.unassign_resource_from_observatory_org(instrument_site_id, org_id) # verify that InstrumentSite is linked to Org assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.InstrumentSite, id_only=True ) self.assertEqual(len(assocs), 0) # remove the InstrumentSite self.OMS.delete_instrument_site(instrument_site_id) assocs, _ = self.RR.find_objects(platform_site_id, PRED.hasSite, RT.InstrumentSite, id_only=True ) self.assertEqual(len(assocs), 1) #todo: remove the dangling association # remove the PlatformSite as a resource of this Observatory self.OMS.unassign_resource_from_observatory_org(platform_site_id, org_id) # verify that PlatformSite is linked to Org assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.PlatformSite, id_only=True ) self.assertEqual(len(assocs), 0) # remove the Site as a resource of this Observatory self.OMS.unassign_resource_from_observatory_org(subsite_id, org_id) # verify that Site is linked to Org assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.Subsite, id_only=True ) self.assertEqual(len(assocs), 0) self.RR.delete(org_id) self.OMS.force_delete_observatory(observatory_id) self.OMS.force_delete_subsite(subsite_id) self.OMS.force_delete_platform_site(platform_site_id) self.OMS.force_delete_instrument_site(instrument_site_id) #@unittest.skip("in development...") @attr('EXT') def test_observatory_org_extended(self): stuff = self._make_associations() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.damsclient.assign_data_product(input_resource_id=stuff.instrument_device_id, data_product_id=data_product_id1) #-------------------------------------------------------------------------------- # Get the extended Site (platformSite) #-------------------------------------------------------------------------------- extended_site = self.OMS.get_site_extension(stuff.platform_site_id) log.debug("extended_site: %s ", str(extended_site)) self.assertEqual(1, len(extended_site.platform_devices)) self.assertEqual(1, len(extended_site.platform_models)) self.assertEqual(stuff.platform_device_id, extended_site.platform_devices[0]._id) self.assertEqual(stuff.platform_model_id, extended_site.platform_models[0]._id) #-------------------------------------------------------------------------------- # Get the extended Org #-------------------------------------------------------------------------------- #test the extended resource extended_org = self.org_management_service.get_marine_facility_extension(stuff.org_id) log.debug("test_observatory_org_extended: extended_org: %s ", str(extended_org)) #self.assertEqual(2, len(extended_org.instruments_deployed) ) #self.assertEqual(1, len(extended_org.platforms_not_deployed) ) self.assertEqual(2, extended_org.number_of_platforms) self.assertEqual(2, len(extended_org.platform_models) ) self.assertEqual(2, extended_org.number_of_instruments) self.assertEqual(2, len(extended_org.instrument_models) ) #test the extended resource of the ION org ion_org_id = self.org_management_service.find_org() extended_org = self.org_management_service.get_marine_facility_extension(ion_org_id._id, user_id=12345) log.debug("test_observatory_org_extended: extended_ION_org: %s ", str(extended_org)) self.assertEqual(0, len(extended_org.members)) self.assertEqual(0, extended_org.number_of_platforms) #self.assertEqual(1, len(extended_org.sites)) #-------------------------------------------------------------------------------- # Get the extended Site #-------------------------------------------------------------------------------- #create device state events to use for op /non-op filtering in extended t = get_ion_ts() self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent', origin = stuff.instrument_device_id, state=ResourceAgentState.STREAMING ) self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent', origin = stuff.instrument_device2_id, state=ResourceAgentState.INACTIVE ) extended_site = self.OMS.get_site_extension(stuff.instrument_site2_id) log.debug("test_observatory_org_extended: extended_site: %s ", str(extended_site)) self.dpclient.delete_data_product(data_product_id1)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient() self.rrclient = ResourceRegistryServiceClient() self.damsclient = DataAcquisitionManagementServiceClient() self.pubsubcli = PubsubManagementServiceClient() self.ingestclient = IngestionManagementServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ self.stream_def_id = self.pubsubcli.create_stream_definition( name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition( name='ingestion worker') ingestion_worker_definition.executable = { 'module': 'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class': 'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition( process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process( self.process_definitions['ingestion_worker'], configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.process_dispatcher.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) IngestionManagementIntTest.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore @attr('EXT') @attr('PREP') def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0 dp_obj.ooi_product_name = "PRODNAME" #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product=dp_obj, stream_definition_id=ctd_stream_def_id) # Assert that the data product has an associated stream at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertNotEquals(len(stream_ids), 0) # Assert that the data product has an associated stream def at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True) self.assertNotEquals(len(stream_ids), 0) self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Created data product %s', dp_obj) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp') dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id2) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) group_names = self.dpsc_cli.get_data_product_group_list() self.assertIn("PRODNAME", group_names) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0 # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description, 'the very first dp') self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Updated data product %s', dp_obj) #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) self.assertEqual(dp_id, extended_product._id) self.assertEqual( ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual( 0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) def ion_object_encoder(obj): return obj.__dict__ #test prepare for create data_product_data = self.dpsc_cli.prepare_data_product_support() #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, "") self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual( len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual( len(data_product_data.associations['Dataset'].resources), 0) self.assertEqual( len(data_product_data.associations['StreamDefinition']. associated_resources), 0) self.assertEqual( len(data_product_data.associations['Dataset'].associated_resources ), 0) #test prepare for update data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id) #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, dp_id) self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual( len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual( len(data_product_data.associations['Dataset'].resources), 1) self.assertEqual( len(data_product_data.associations['StreamDefinition']. associated_resources), 1) self.assertEqual( data_product_data.associations['StreamDefinition']. associated_resources[0].s, dp_id) self.assertEqual( len(data_product_data.associations['Dataset'].associated_resources ), 1) self.assertEqual( data_product_data.associations['Dataset'].associated_resources[0]. s, dp_id) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) # Assert that there are no associated streams leftover after deleting the data product stream_ids, assoc_ids = self.rrclient.find_objects( dp_id, PRED.hasStream, RT.Stream, True) self.assertEquals(len(stream_ids), 0) self.assertEquals(len(assoc_ids), 0) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product(dp_id) # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) self.assertTrue(len(events) > 0) def test_data_product_stream_def(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='Simulated CTD data', parameter_dictionary_id=pdict_id) dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') dp_id = self.dpsc_cli.create_data_product( data_product=dp_obj, stream_definition_id=ctd_stream_def_id) stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id) self.assertEquals(ctd_stream_def_id, stream_def_id) def test_derived_data_product(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='ctd parsed', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id) dp = DataProduct(name='Instrument DP') dp_id = self.dpsc_cli.create_data_product( dp, stream_definition_id=ctd_stream_def_id) self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id) self.dpsc_cli.activate_data_product_persistence(dp_id) self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Make the derived data product simple_stream_def_id = self.pubsubcli.create_stream_definition( name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time', 'temp']) tempwat_dp = DataProduct(name='TEMPWAT', category=DataProductTypeEnum.DERIVED) tempwat_dp_id = self.dpsc_cli.create_data_product( tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id) self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id) # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) rdt['pressure'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id, route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True) tempwat_dataset_id = tempwat_dataset_ids[0] granule = self.data_retriever.retrieve( tempwat_dataset_id, delivery_format=simple_stream_def_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['time'], np.arange(20)) self.assertEquals(set(rdt.fields), set(['time', 'temp'])) def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Construct temporal and spatial Coordinate Reference System objects dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product=dp_obj, stream_definition_id=ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # Subscribe to persist events #------------------------------------------------------------------------------------------------ queue = gevent.queue.Queue() def info_event_received(message, headers): queue.put(message) es = EventSubscriber(event_type=OT.InformationContentStatusEvent, callback=info_event_received, origin=dp_id, auto_delete=True) es.start() self.addCleanup(es.stop) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id, route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_ids[0]) self.assertIsInstance(replay_data, Granule) log.debug( "The data retriever was able to replay the dataset that was attached to the data product " "we wanted to be persisted. Therefore the data product was indeed persisted with " "otherwise we could not have retrieved its dataset using the data retriever. Therefore " "this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'" ) data_product_object = self.rrclient.read(dp_id) self.assertEquals(data_product_object.name, 'DP1') self.assertEquals(data_product_object.description, 'some new dp') log.debug( "Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. " " Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the " "resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description, data_product_object.name, data_product_object.description)) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) dataset_modified.clear() rdt['time'] = np.arange(20, 40) publisher.publish(rdt.to_granule()) self.assertFalse(dataset_modified.wait(2)) self.dpsc_cli.activate_data_product_persistence(dp_id) dataset_modified.clear() publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_almost_equal(rdt['time'], np.arange(40)) dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True) self.assertEquals(len(dataset_ids), 1) self.dpsc_cli.suspend_data_product_persistence(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.rrclient.read(dp_id) info_event_counter = 0 runtime = 0 starttime = time.time() caught_events = [] #check that the four InfoStatusEvents were received while info_event_counter < 4 and runtime < 60: a = queue.get(timeout=60) caught_events.append(a) info_event_counter += 1 runtime = time.time() - starttime self.assertEquals(info_event_counter, 4)
class TestGranulePublish(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id, configuration=configuration) return pid #overriding trigger function here to use new granule def test_granule_publish(self): log.debug("test_granule_publish ") self.loggerpids = [] #retrieve the param dict from the repository pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) stream_definition_id = self.pubsubclient.create_stream_definition('parsed stream', parameter_dictionary_id=pdict_id) dp_obj = IonObject(RT.DataProduct, name=str(uuid.uuid4()), description='ctd stream test') data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=stream_definition_id) # Retrieve the id of the output stream of the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'test_granule_publish: Data product streams1 = %s', stream_ids) pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id) #create the publisher from the stream route stream_route = self.pubsubclient.read_stream_route(stream_ids[0]) publisher = StandaloneStreamPublisher(stream_ids[0], stream_route) # this is one sample from the ctd driver tomato = {"driver_timestamp": 3555971105.1268806, "instrument_id": "ABC-123", "pkt_format_id": "JSON_Data", "pkt_version": 1, "preferred_timestamp": "driver_timestamp", "quality_flag": "ok", "stream_name": "parsed", "values": [{"value": 22.9304, "value_id": "temp"}, {"value": 51.57381, "value_id": "conductivity"}, {"value": 915.551, "value_id": "pressure"}]} for value in tomato['values']: log.debug("test_granule_publish: Looping tomato values key: %s val: %s ", str(value['value']), str(value['value_id'])) if value['value_id'] in rdt: rdt[value['value_id']] = numpy.array( [ value['value'] ] ) log.debug("test_granule_publish: Added data item %s val: %s ", str(value['value']), str(value['value_id']) ) g = rdt.to_granule() publisher.publish(g) gevent.sleep(3) for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) #-------------------------------------------------------------------------------- # Cleanup data products #-------------------------------------------------------------------------------- dp_ids, _ = self.rrclient.find_resources(restype=RT.DataProduct, id_only=True) for dp_id in dp_ids: self.dataproductclient.delete_data_product(dp_id)
class TestActivateRSNVel3DInstrument(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateRSNVel3DInstrument, self).setUp() config = DotDict() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid @attr('LOCOINT') @unittest.skip('under construction') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint': {'receive': {'timeout': 180}}}) def test_activate_rsn_vel3d(self): log.info( "--------------------------------------------------------------------------------------------------------" ) # load_parameter_scenarios self.container.spawn_process( "Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="BETA", path="master", categories= "ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition", clearcols="owner_id,org_ids", assets="res/preload/r2_ioc/ooi_assets", parseooi="True", )) self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='Vel3DMModel', description="Vel3DMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug('test_activate_rsn_vel3d new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw') vel3d_b_sample = StreamConfiguration( stream_name='vel3d_b_sample', parameter_dictionary_name='vel3d_b_sample') vel3d_b_engineering = StreamConfiguration( stream_name='vel3d_b_engineering', parameter_dictionary_name='vel3d_b_engineering') RSN_VEL3D_01 = { 'DEV_ADDR': "10.180.80.6", 'DEV_PORT': 2101, 'DATA_PORT': 1026, 'CMD_PORT': 1025, 'PA_BINARY': "port_agent" } # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='Vel3DAgent', description="Vel3DAgent", driver_uri= "http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg", stream_configurations=[ raw_config, vel3d_b_sample, vel3d_b_engineering ]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('test_activate_rsn_vel3d new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activate_rsn_vel3d: Create instrument resource to represent the Vel3D ' ) instDevice_obj = IonObject(RT.InstrumentDevice, name='Vel3DDevice', description="Vel3DDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug("test_activate_rsn_vel3d: new InstrumentDevice id = %s ", instDevice_id) port_agent_config = { 'device_addr': '10.180.80.6', 'device_port': 2101, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': 1025, 'data_port': 1026, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='Vel3DAgentInstance', description="Vel3DAgentInstance", port_agent_config=port_agent_config, alerts=[]) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) parsed_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'vel3d_b_sample', id_only=True) parsed_sample_stream_def_id = self.pubsubcli.create_stream_definition( name='vel3d_b_sample', parameter_dictionary_id=parsed_sample_pdict_id) parsed_eng_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'vel3d_b_engineering', id_only=True) parsed_eng_stream_def_id = self.pubsubcli.create_stream_definition( name='vel3d_b_engineering', parameter_dictionary_id=parsed_eng_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'raw', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='vel3d_b_sample', description='vel3d_b_sample') sample_data_product_id = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_sample_stream_def_id) log.debug('new dp_id = %s', sample_data_product_id) self.dpclient.activate_data_product_persistence( data_product_id=sample_data_product_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=sample_data_product_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasStream, None, True) log.debug('sample_data_product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for sample_data_product = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('vel3d_b_sample', stream_ids[0]) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='vel3d_b_engineering', description='vel3d_b_engineering') eng_data_product_id = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_eng_stream_def_id) log.debug('new dp_id = %s', eng_data_product_id) self.dpclient.activate_data_product_persistence( data_product_id=eng_data_product_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=eng_data_product_id) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test') data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('test_activate_rsn_vel3d Data product streams2 = %s', str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('test_activate_rsn_vel3d Data set for data_product_id2 = %s', dataset_ids[0]) self.raw_dataset = dataset_ids[0] def start_instrument_agent(): self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) #log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) def check_state(label, desired_state): actual_state = self._ia_client.get_agent_state() log.debug("%s instrument agent is in state '%s'", label, actual_state) self.assertEqual(desired_state, actual_state) log.debug("test_activate_rsn_vel3d: got ia client %s", str(self._ia_client)) check_state("just-spawned", ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: initialize %s", str(retval)) check_state("initialized", ResourceAgentState.INACTIVE) log.debug("test_activate_rsn_vel3d Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: return value from go_active %s", str(reply)) check_state("activated", ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("current state after sending go_active command %s", str(state)) # cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: run %s", str(reply)) check_state("commanded", ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("current state after sending run command %s", str(state)) # cmd = AgentCommand(command=ProtocolEvent.START_AUTOSAMPLE) # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # gevent.sleep(5) # # cmd = AgentCommand(command=ProtocolEvent.STOP_AUTOSAMPLE) # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) # retval = self._ia_client.execute_agent(cmd) # state = retval.result # log.debug("current state after sending STOP_AUTOSAMPLE command %s" , str(state)) # # cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.STOPPED, state) # # cmd = AgentCommand(command=ResourceAgentEvent.RESUME) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.IDLE, state) # # cmd = AgentCommand(command=ResourceAgentEvent.RUN) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) log.debug("test_activate_rsn_vel3d: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: return from reset %s", str(reply)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data_raw, Granule) rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw) log.debug("RDT raw: %s", str(rdt_raw.pretty_print())) self.assertIn('raw', rdt_raw) raw_vals = rdt_raw['raw'] #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(sample_data_product_id) self.dpclient.delete_data_product(eng_data_product_id) self.dpclient.delete_data_product(data_product_id2)
class TestObservatoryManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.org_management_service = OrgManagementServiceClient( node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #print 'TestObservatoryManagementServiceIntegration: started services' self.event_publisher = EventPublisher() # @unittest.skip('this exists only for debugging the launch process') # def test_just_the_setup(self): # return def destroy(self, resource_ids): self.OMS.force_delete_observatory(resource_ids.observatory_id) self.OMS.force_delete_subsite(resource_ids.subsite_id) self.OMS.force_delete_subsite(resource_ids.subsite2_id) self.OMS.force_delete_subsite(resource_ids.subsiteb_id) self.OMS.force_delete_subsite(resource_ids.subsitez_id) self.OMS.force_delete_platform_site(resource_ids.platform_site_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb2_id) self.OMS.force_delete_platform_site(resource_ids.platform_site3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site2_id) self.OMS.force_delete_instrument_site( resource_ids.instrument_siteb3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site4_id) #@unittest.skip('targeting') def test_observatory_management(self): resources = self._make_associations() self._do_test_find_related_sites(resources) self._do_test_get_sites_devices_status(resources) self._do_test_find_site_data_products(resources) self._do_test_find_related_frames_of_reference(resources) self._do_test_create_geospatial_point_center(resources) self._do_test_find_observatory_org(resources) self.destroy(resources) def _do_test_find_related_sites(self, resources): site_resources, site_children, _, _ = self.OMS.find_related_sites( resources.org_id) #import sys, pprint #print >> sys.stderr, pprint.pformat(site_resources) #print >> sys.stderr, pprint.pformat(site_children) #self.assertIn(resources.org_id, site_resources) self.assertIn(resources.observatory_id, site_resources) self.assertIn(resources.subsite_id, site_resources) self.assertIn(resources.subsite_id, site_resources) self.assertIn(resources.subsite2_id, site_resources) self.assertIn(resources.platform_site_id, site_resources) self.assertIn(resources.instrument_site_id, site_resources) self.assertEquals(len(site_resources), 13) self.assertEquals(site_resources[resources.observatory_id].type_, RT.Observatory) self.assertIn(resources.org_id, site_children) self.assertIn(resources.observatory_id, site_children) self.assertIn(resources.subsite_id, site_children) self.assertIn(resources.subsite_id, site_children) self.assertIn(resources.subsite2_id, site_children) self.assertIn(resources.platform_site_id, site_children) self.assertNotIn(resources.instrument_site_id, site_children) self.assertEquals(len(site_children), 9) self.assertIsInstance(site_children[resources.subsite_id], list) self.assertEquals(len(site_children[resources.subsite_id]), 2) def _do_test_get_sites_devices_status(self, resources): #bin/nosetests -s -v --nologcapture ion/services/sa/observatory/test/test_observatory_management_service_integration.py:TestObservatoryManagementServiceIntegration.test_observatory_management full_result_dict = self.OMS.get_sites_devices_status( parent_resource_ids=[resources.org_id], include_sites=True) result_dict = full_result_dict[resources.org_id] site_resources = result_dict.get("site_resources", None) site_children = result_dict.get("site_children", None) self.assertEquals(len(site_resources), 14) self.assertEquals(len(site_children), 9) full_result_dict = self.OMS.get_sites_devices_status( parent_resource_ids=[resources.org_id], include_sites=True, include_devices=True, include_status=True) result_dict = full_result_dict[resources.org_id] log.debug("RESULT DICT: %s", result_dict.keys()) site_resources = result_dict.get("site_resources", None) site_children = result_dict.get("site_children", None) site_status = result_dict.get("site_status", None) self.assertEquals(len(site_resources), 14) self.assertEquals(len(site_children), 9) full_result_dict = self.OMS.get_sites_devices_status( parent_resource_ids=[resources.observatory_id], include_sites=True, include_devices=True, include_status=True) result_dict = full_result_dict[resources.observatory_id] site_resources = result_dict.get("site_resources") site_children = result_dict.get("site_children") site_status = result_dict.get("site_status") self.assertEquals(len(site_resources), 13) self.assertEquals(len(site_children), 8) def _do_test_find_site_data_products(self, resources): res_dict = self.OMS.find_site_data_products(resources.org_id) #import sys, pprint #print >> sys.stderr, pprint.pformat(res_dict) self.assertIsNone(res_dict['data_product_resources']) self.assertIn(resources.platform_device_id, res_dict['device_data_products']) self.assertIn(resources.instrument_device_id, res_dict['device_data_products']) #@unittest.skip('targeting') def _do_test_find_related_frames_of_reference(self, stuff): # finding subordinates gives a dict of obj lists, convert objs to ids def idify(adict): ids = {} for k, v in adict.iteritems(): ids[k] = [] for obj in v: ids[k].append(obj._id) return ids # a short version of the function we're testing, with id-ify def short(resource_id, output_types): ret = self.OMS.find_related_frames_of_reference( resource_id, output_types) return idify(ret) #set up associations first stuff = self._make_associations() #basic traversal of tree from instrument to platform ids = short(stuff.instrument_site_id, [RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) #since this is the first search, just make sure the input inst_id got stripped if RT.InstrumentSite in ids: self.assertNotIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #basic traversal of tree from platform to instrument ids = short(stuff.platform_siteb_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) self.assertNotIn(stuff.instrument_site2_id, ids[RT.InstrumentSite]) #full traversal of tree from observatory down to instrument ids = short(stuff.observatory_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #full traversal of tree from instrument to observatory ids = short(stuff.instrument_site_id, [RT.Observatory]) self.assertIn(RT.Observatory, ids) self.assertIn(stuff.observatory_id, ids[RT.Observatory]) #partial traversal, only down to platform ids = short(stuff.observatory_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site3_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(RT.InstrumentSite, ids) #partial traversal, only down to platform ids = short(stuff.instrument_site_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertNotIn(RT.Observatory, ids) self.destroy(stuff) def _make_associations(self): """ create one of each resource and association used by OMS to guard against problems in ion-definitions """ #raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41") """ the tree we're creating (observatory, sites, platforms, instruments) rows are lettered, colums numbered. - first row is implied a - first column is implied 1 - site Z, just because O--Sz | S--S2--P3--I4 | Sb-Pb2-Ib3 | P--I2 <- PlatformDevice, InstrumentDevice2 | Pb <- PlatformDevice b | I <- InstrumentDevice """ org_id = self.OMS.create_marine_facility(any_old(RT.Org)) def create_under_org(resource_type, extra_fields=None): obj = any_old(resource_type, extra_fields) if RT.InstrumentDevice == resource_type: resource_id = self.IMS.create_instrument_device(obj) else: resource_id, _ = self.RR.create(obj) self.OMS.assign_resource_to_observatory_org( resource_id=resource_id, org_id=org_id) return resource_id #stuff we control observatory_id = create_under_org(RT.Observatory) subsite_id = create_under_org(RT.Subsite) subsite2_id = create_under_org(RT.Subsite) subsiteb_id = create_under_org(RT.Subsite) subsitez_id = create_under_org(RT.Subsite) platform_site_id = create_under_org(RT.PlatformSite) platform_siteb_id = create_under_org(RT.PlatformSite) platform_siteb2_id = create_under_org(RT.PlatformSite) platform_site3_id = create_under_org(RT.PlatformSite) instrument_site_id = create_under_org(RT.InstrumentSite) instrument_site2_id = create_under_org(RT.InstrumentSite) instrument_siteb3_id = create_under_org(RT.InstrumentSite) instrument_site4_id = create_under_org(RT.InstrumentSite) #stuff we associate to instrument_device_id = create_under_org(RT.InstrumentDevice) instrument_device2_id = create_under_org(RT.InstrumentDevice) platform_device_id = create_under_org(RT.PlatformDevice) platform_deviceb_id = create_under_org(RT.PlatformDevice) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) deployment_id, _ = self.RR.create(any_old(RT.Deployment)) # marine tracking resources asset_id = create_under_org(RT.Asset) asset_type_id = create_under_org(RT.AssetType) event_duration_id = create_under_org(RT.EventDuration) event_duration_type_id = create_under_org(RT.EventDurationType) #observatory self.RR.create_association(observatory_id, PRED.hasSite, subsite_id) self.RR.create_association(observatory_id, PRED.hasSite, subsitez_id) #site self.RR.create_association(subsite_id, PRED.hasSite, subsite2_id) self.RR.create_association(subsite_id, PRED.hasSite, subsiteb_id) self.RR.create_association(subsite2_id, PRED.hasSite, platform_site3_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_siteb2_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_site_id) #platform_site(s) self.RR.create_association(platform_site3_id, PRED.hasSite, instrument_site4_id) self.RR.create_association(platform_siteb2_id, PRED.hasSite, instrument_siteb3_id) self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site2_id) self.RR.create_association(platform_site_id, PRED.hasSite, platform_siteb_id) self.RR.create_association(platform_siteb_id, PRED.hasSite, instrument_site_id) self.RR.create_association(platform_siteb_id, PRED.hasDevice, platform_deviceb_id) #test network parent link self.OMS.assign_device_to_network_parent(platform_device_id, platform_deviceb_id) self.RR.create_association(platform_site_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id) self.RR.create_association(platform_site_id, PRED.hasDeployment, deployment_id) #instrument_site(s) self.RR.create_association(instrument_site_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(instrument_site_id, PRED.hasDeployment, deployment_id) self.RR.create_association(instrument_site2_id, PRED.hasDevice, instrument_device2_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id) ret = DotDict() ret.org_id = org_id ret.observatory_id = observatory_id ret.subsite_id = subsite_id ret.subsite2_id = subsite2_id ret.subsiteb_id = subsiteb_id ret.subsitez_id = subsitez_id ret.platform_site_id = platform_site_id ret.platform_siteb_id = platform_siteb_id ret.platform_siteb2_id = platform_siteb2_id ret.platform_site3_id = platform_site3_id ret.instrument_site_id = instrument_site_id ret.instrument_site2_id = instrument_site2_id ret.instrument_siteb3_id = instrument_siteb3_id ret.instrument_site4_id = instrument_site4_id ret.instrument_device_id = instrument_device_id ret.instrument_device2_id = instrument_device2_id ret.platform_device_id = platform_device_id ret.platform_deviceb_id = platform_deviceb_id ret.instrument_model_id = instrument_model_id ret.platform_model_id = platform_model_id ret.deployment_id = deployment_id ret.asset_id = asset_id ret.asset_type_id = asset_type_id ret.event_duration_id = event_duration_id ret.event_duration_type_id = event_duration_type_id return ret #@unittest.skip("targeting") def test_create_observatory(self): observatory_obj = IonObject(RT.Observatory, name='TestFacility', description='some new mf') observatory_id = self.OMS.create_observatory(observatory_obj) self.OMS.force_delete_observatory(observatory_id) #@unittest.skip("targeting") def _do_test_create_geospatial_point_center(self, resources): platformsite_obj = IonObject(RT.PlatformSite, name='TestPlatformSite', description='some new TestPlatformSite') geo_index_obj = IonObject(OT.GeospatialBounds) geo_index_obj.geospatial_latitude_limit_north = 20.0 geo_index_obj.geospatial_latitude_limit_south = 10.0 geo_index_obj.geospatial_longitude_limit_east = 15.0 geo_index_obj.geospatial_longitude_limit_west = 20.0 platformsite_obj.constraint_list = [geo_index_obj] platformsite_id = self.OMS.create_platform_site(platformsite_obj) # now get the dp back to see if it was updated platformsite_obj = self.OMS.read_platform_site(platformsite_id) self.assertEquals('some new TestPlatformSite', platformsite_obj.description) self.assertAlmostEqual(15.0, platformsite_obj.geospatial_point_center.lat, places=1) #now adjust a few params platformsite_obj.description = 'some old TestPlatformSite' geo_index_obj = IonObject(OT.GeospatialBounds) geo_index_obj.geospatial_latitude_limit_north = 30.0 geo_index_obj.geospatial_latitude_limit_south = 20.0 platformsite_obj.constraint_list = [geo_index_obj] update_result = self.OMS.update_platform_site(platformsite_obj) # now get the dp back to see if it was updated platformsite_obj = self.OMS.read_platform_site(platformsite_id) self.assertEquals('some old TestPlatformSite', platformsite_obj.description) self.assertAlmostEqual(25.0, platformsite_obj.geospatial_point_center.lat, places=1) self.OMS.force_delete_platform_site(platformsite_id) #@unittest.skip("targeting") def _do_test_find_observatory_org(self, resources): log.debug("Make TestOrg") org_obj = IonObject(RT.Org, name='TestOrg', description='some new mf org') org_id = self.OMS.create_marine_facility(org_obj) log.debug("Make Observatory") observatory_obj = IonObject(RT.Observatory, name='TestObservatory', description='some new obs') observatory_id = self.OMS.create_observatory(observatory_obj) log.debug("assign observatory to org") self.OMS.assign_resource_to_observatory_org(observatory_id, org_id) log.debug("verify assigment") org_objs = self.OMS.find_org_by_observatory(observatory_id) self.assertEqual(1, len(org_objs)) self.assertEqual(org_id, org_objs[0]._id) log.debug("org_id=<" + org_id + ">") log.debug("create a subsite with parent Observatory") subsite_obj = IonObject(RT.Subsite, name='TestSubsite', description='sample subsite') subsite_id = self.OMS.create_subsite(subsite_obj, observatory_id) self.assertIsNotNone(subsite_id, "Subsite not created.") log.debug("verify that Subsite is linked to Observatory") mf_subsite_assoc = self.RR.get_association(observatory_id, PRED.hasSite, subsite_id) self.assertIsNotNone(mf_subsite_assoc, "Subsite not connected to Observatory.") log.debug("add the Subsite as a resource of this Observatory") self.OMS.assign_resource_to_observatory_org(resource_id=subsite_id, org_id=org_id) log.debug("verify that Subsite is linked to Org") org_subsite_assoc = self.RR.get_association(org_id, PRED.hasResource, subsite_id) self.assertIsNotNone(org_subsite_assoc, "Subsite not connected as resource to Org.") log.debug("create a logical platform with parent Subsite") platform_site_obj = IonObject(RT.PlatformSite, name='TestPlatformSite', description='sample logical platform') platform_site_id = self.OMS.create_platform_site( platform_site_obj, subsite_id) self.assertIsNotNone(platform_site_id, "PlatformSite not created.") log.debug("verify that PlatformSite is linked to Site") site_lp_assoc = self.RR.get_association(subsite_id, PRED.hasSite, platform_site_id) self.assertIsNotNone(site_lp_assoc, "PlatformSite not connected to Site.") log.debug("add the PlatformSite as a resource of this Observatory") self.OMS.assign_resource_to_observatory_org( resource_id=platform_site_id, org_id=org_id) log.debug("verify that PlatformSite is linked to Org") org_lp_assoc = self.RR.get_association(org_id, PRED.hasResource, platform_site_id) self.assertIsNotNone(org_lp_assoc, "PlatformSite not connected as resource to Org.") log.debug("create a logical instrument with parent logical platform") instrument_site_obj = IonObject( RT.InstrumentSite, name='TestInstrumentSite', description='sample logical instrument') instrument_site_id = self.OMS.create_instrument_site( instrument_site_obj, platform_site_id) self.assertIsNotNone(instrument_site_id, "InstrumentSite not created.") log.debug("verify that InstrumentSite is linked to PlatformSite") li_lp_assoc = self.RR.get_association(platform_site_id, PRED.hasSite, instrument_site_id) self.assertIsNotNone(li_lp_assoc, "InstrumentSite not connected to PlatformSite.") log.debug("add the InstrumentSite as a resource of this Observatory") self.OMS.assign_resource_to_observatory_org( resource_id=instrument_site_id, org_id=org_id) log.debug("verify that InstrumentSite is linked to Org") org_li_assoc = self.RR.get_association(org_id, PRED.hasResource, instrument_site_id) self.assertIsNotNone( org_li_assoc, "InstrumentSite not connected as resource to Org.") log.debug( "remove the InstrumentSite as a resource of this Observatory") self.OMS.unassign_resource_from_observatory_org( instrument_site_id, org_id) log.debug("verify that InstrumentSite is linked to Org") assocs, _ = self.RR.find_objects(org_id, PRED.hasResource, RT.InstrumentSite, id_only=True) self.assertEqual(0, len(assocs)) log.debug( "remove the InstrumentSite, association should drop automatically") self.OMS.delete_instrument_site(instrument_site_id) assocs, _ = self.RR.find_objects(platform_site_id, PRED.hasSite, RT.InstrumentSite, id_only=True) self.assertEqual(0, len(assocs)) log.debug("remove the PlatformSite as a resource of this Observatory") self.OMS.unassign_resource_from_observatory_org( platform_site_id, org_id) log.debug("verify that PlatformSite is linked to Org") assocs, _ = self.RR.find_objects(org_id, PRED.hasResource, RT.PlatformSite, id_only=True) self.assertEqual(0, len(assocs)) log.debug("remove the Site as a resource of this Observatory") self.OMS.unassign_resource_from_observatory_org(subsite_id, org_id) log.debug("verify that Site is linked to Org") assocs, _ = self.RR.find_objects(org_id, PRED.hasResource, RT.Subsite, id_only=True) self.assertEqual(0, len(assocs)) self.RR.delete(org_id) self.OMS.force_delete_observatory(observatory_id) self.OMS.force_delete_subsite(subsite_id) self.OMS.force_delete_platform_site(platform_site_id) self.OMS.force_delete_instrument_site(instrument_site_id) @attr('EXT') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Skip test while in CEI LAUNCH mode as it depends on modifying CFG on service side' ) def test_observatory_extensions(self): self.patch_cfg(CFG["container"], {"extended_resources": { "strip_results": False }}) obs_id = self.RR2.create(any_old(RT.Observatory)) pss_id = self.RR2.create( any_old(RT.PlatformSite, dict(alt_resource_type="StationSite"))) pas_id = self.RR2.create( any_old(RT.PlatformSite, dict(alt_resource_type="PlatformAssemblySite"))) pcs_id = self.RR2.create( any_old(RT.PlatformSite, dict(alt_resource_type="PlatformComponentSite"))) ins_id = self.RR2.create(any_old(RT.InstrumentSite)) obs_obj = self.RR2.read(obs_id) pss_obj = self.RR2.read(pss_id) pas_obj = self.RR2.read(pas_id) pcs_obj = self.RR2.read(pcs_id) ins_obj = self.RR2.read(ins_id) self.RR2.create_association(obs_id, PRED.hasSite, pss_id) self.RR2.create_association(pss_id, PRED.hasSite, pas_id) self.RR2.create_association(pas_id, PRED.hasSite, pcs_id) self.RR2.create_association(pcs_id, PRED.hasSite, ins_id) extended_obs = self.OMS.get_observatory_site_extension(obs_id, user_id=12345) self.assertEqual([pss_obj], extended_obs.platform_station_sites) self.assertEqual([pas_obj], extended_obs.platform_assembly_sites) self.assertEqual([pcs_obj], extended_obs.platform_component_sites) self.assertEqual([ins_obj], extended_obs.instrument_sites) extended_pss = self.OMS.get_observatory_site_extension(obs_id, user_id=12345) self.assertEqual([pas_obj], extended_pss.platform_assembly_sites) self.assertEqual([pcs_obj], extended_pss.platform_component_sites) self.assertEqual([ins_obj], extended_pss.instrument_sites) extended_pas = self.OMS.get_observatory_site_extension(pas_id, user_id=12345) self.assertEqual([pcs_obj], extended_pas.platform_component_sites) self.assertEqual([ins_obj], extended_pas.instrument_sites) extended_pcs = self.OMS.get_platform_component_site_extension( pcs_id, user_id=12345) self.assertEqual([ins_obj], extended_pcs.instrument_sites) #@unittest.skip("in development...") @attr('EXT') @attr('EXT1') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Skip test while in CEI LAUNCH mode as it depends on modifying CFG on service side' ) def test_observatory_org_extended(self): self.patch_cfg(CFG["container"], {"extended_resources": { "strip_results": False }}) stuff = self._make_associations() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test') data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.damsclient.assign_data_product( input_resource_id=stuff.instrument_device_id, data_product_id=data_product_id1) #Create a user to be used as regular member member_actor_obj = IonObject(RT.ActorIdentity, name='org member actor') member_actor_id, _ = self.RR.create(member_actor_obj) assert (member_actor_id) member_actor_header = get_actor_header(member_actor_id) member_user_obj = IonObject(RT.UserInfo, name='org member user') member_user_id, _ = self.RR.create(member_user_obj) assert (member_user_id) self.RR.create_association(subject=member_actor_id, predicate=PRED.hasInfo, object=member_user_id) #Build the Service Agreement Proposal to enroll a user actor sap = IonObject(OT.EnrollmentProposal, consumer=member_actor_id, provider=stuff.org_id) sap_response = self.org_management_service.negotiate( sap, headers=member_actor_header) #enroll the member without using negotiation self.org_management_service.enroll_member(org_id=stuff.org_id, actor_id=member_actor_id) #-------------------------------------------------------------------------------- # Get the extended Site (platformSite) #-------------------------------------------------------------------------------- try: extended_site = self.OMS.get_site_extension(stuff.platform_site_id) except: log.error('failed to get extended site', exc_info=True) raise log.debug("extended_site: %r ", extended_site) self.assertEquals(stuff.subsiteb_id, extended_site.parent_site._id) self.assertEqual(2, len(extended_site.sites)) self.assertEqual(2, len(extended_site.platform_devices)) self.assertEqual(2, len(extended_site.platform_models)) self.assertIn(stuff.platform_device_id, [o._id for o in extended_site.platform_devices]) self.assertIn( stuff.platform_model_id, [o._id for o in extended_site.platform_models if o is not None]) log.debug( "verify that PlatformDeviceb is linked to PlatformDevice with hasNetworkParent link" ) associations = self.RR.find_associations( subject=stuff.platform_deviceb_id, predicate=PRED.hasNetworkParent, object=stuff.platform_device_id, id_only=True) self.assertIsNotNone( associations, "PlatformDevice child not connected to PlatformDevice parent.") #-------------------------------------------------------------------------------- # Get the extended Org #-------------------------------------------------------------------------------- #test the extended resource extended_org = self.OMS.get_marine_facility_extension(stuff.org_id) log.debug("test_observatory_org_extended: extended_org: %s ", str(extended_org)) #self.assertEqual(2, len(extended_org.instruments_deployed) ) #self.assertEqual(1, len(extended_org.platforms_not_deployed) ) self.assertEqual(2, extended_org.number_of_platforms) self.assertEqual(2, len(extended_org.platform_models)) self.assertEqual(2, extended_org.number_of_instruments) self.assertEqual(2, len(extended_org.instrument_models)) self.assertEqual(1, len(extended_org.members)) self.assertNotEqual(extended_org.members[0]._id, member_actor_id) self.assertEqual(extended_org.members[0]._id, member_user_id) self.assertEqual(1, len(extended_org.open_requests)) self.assertTrue(len(extended_site.deployments) > 0) self.assertEqual(len(extended_site.deployments), len(extended_site.deployment_info)) self.assertEqual(1, extended_org.number_of_assets) self.assertEqual(1, extended_org.number_of_asset_types) self.assertEqual(1, extended_org.number_of_event_durations) self.assertEqual(1, extended_org.number_of_event_duration_types) #test the extended resource of the ION org ion_org_id = self.org_management_service.find_org() extended_org = self.OMS.get_marine_facility_extension(ion_org_id._id, user_id=12345) log.debug("test_observatory_org_extended: extended_ION_org: %s ", str(extended_org)) self.assertEqual(1, len(extended_org.members)) self.assertEqual(0, extended_org.number_of_platforms) #self.assertEqual(1, len(extended_org.sites)) #-------------------------------------------------------------------------------- # Get the extended Site #-------------------------------------------------------------------------------- #create device state events to use for op /non-op filtering in extended t = get_ion_ts() self.event_publisher.publish_event( ts_created=t, event_type='ResourceAgentStateEvent', origin=stuff.instrument_device_id, state=ResourceAgentState.STREAMING) self.event_publisher.publish_event( ts_created=t, event_type='ResourceAgentStateEvent', origin=stuff.instrument_device2_id, state=ResourceAgentState.INACTIVE) extended_site = self.OMS.get_site_extension(stuff.instrument_site2_id) log.debug("test_observatory_org_extended: extended_site: %s ", str(extended_site)) self.dpclient.delete_data_product(data_product_id1)
class TestDataProductVersions(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.client = DataProductManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #@unittest.skip('not working') def test_createDataProductVersionSimple(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='test', parameter_dictionary_id=pdict_id) # test creating a new data product which will also create the initial/default version log.debug('Creating new data product with a stream definition') # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) dp_id = self.client.create_data_product(dp_obj, ctd_stream_def_id) log.debug( 'new dp_id = %s', str(dp_id)) dpc_id = self.client.create_data_product_collection( data_product_id=dp_id, collection_name='firstCollection', collection_description='collection desc') #test that the links exist version_ids, _ = self.rrclient.find_objects(subject=dpc_id, predicate=PRED.hasVersion, id_only=True) log.debug( 'version_ids = %s', str(version_ids)) self.assertTrue(version_ids, 'Failed to connect the data product to the version collection.') self.assertTrue(version_ids[0] == dp_id, 'Failed to connect the data product to the version collection.') # test creating a subsequent data product version which will update the data product pointers dp2_obj = IonObject(RT.DataProduct, name='DP2', description='a second dp', temporal_domain = tdom, spatial_domain = sdom) dp2_id = self.client.create_data_product(dp2_obj, ctd_stream_def_id) log.debug( 'second dp_id = %s', str(dp2_id)) self.client.add_data_product_version_to_collection(data_product_id=dp2_id, data_product_collection_id=dpc_id, version_name="second version", version_description="desc" ) #test that the links exist version_ids, _ = self.rrclient.find_objects(subject=dpc_id, predicate=PRED.hasVersion, id_only=True) if len(version_ids) != 2: self.fail("data product should have two versions") recent_version_id = self.client.get_current_version(dpc_id) self.assertEquals(recent_version_id, dp2_id ) base_version_id = self.client.get_base_version(dpc_id) self.assertEquals(base_version_id, dp_id ) self.client.delete_data_product(dp_id) self.client.delete_data_product(dp2_id) self.client.delete_data_product_collection(dpc_id) self.client.force_delete_data_product_collection(dpc_id) # now try to get the deleted dp object try: dp_obj = self.client.read_data_product_collection(dpc_id) except NotFound as ex: pass else: self.fail("deleted data product collection was found during read")
class TestDataProductManagementServiceCoverage(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() log.debug("Start rel from url") self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.DPMS = DataProductManagementServiceClient() self.RR = ResourceRegistryServiceClient() self.RR2 = EnhancedResourceRegistryClient(self.RR) self.DAMS = DataAcquisitionManagementServiceClient() self.PSMS = PubsubManagementServiceClient() self.ingestclient = IngestionManagementServiceClient() self.PD = ProcessDispatcherServiceClient() self.DSMS = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ log.debug("get datastore") datastore_name = CACHE_DATASTORE_NAME self.db = self.container.datastore_manager.get_datastore(datastore_name) self.stream_def_id = self.PSMS.create_stream_definition(name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.PD.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] self.addCleanup(self.cleaning_up) @staticmethod def clean_subscriptions(): ingestion_management = IngestionManagementServiceClient() pubsub = PubsubManagementServiceClient() rr = ResourceRegistryServiceClient() ingestion_config_ids = ingestion_management.list_ingestion_configurations(id_only=True) for ic in ingestion_config_ids: subscription_ids, assocs = rr.find_objects(subject=ic, predicate=PRED.hasSubscription, id_only=True) for subscription_id, assoc in zip(subscription_ids, assocs): rr.delete_association(assoc) try: pubsub.deactivate_subscription(subscription_id) except: log.exception("Unable to decativate subscription: %s", subscription_id) pubsub.delete_subscription(subscription_id) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.PD.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) TestDataProductManagementServiceCoverage.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def test_CRUD_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary = self.DSMS.read_parameter_dictionary_by_name('ctd_parsed_param_dict') ctd_stream_def_id = self.PSMS.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Generic time-series data domain creation tdom, sdom = time_series_domain() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0 dp_obj.ooi_product_name = "PRODNAME" #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ log.debug("create dataset") dataset_id = self.RR2.create(any_old(RT.Dataset)) log.debug("dataset_id = %s", dataset_id) log.debug("create data product 1") dp_id = self.DPMS.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id, dataset_id=dataset_id) log.debug("dp_id = %s", dp_id) # Assert that the data product has an associated stream at this stage stream_ids, _ = self.RR.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertNotEquals(len(stream_ids), 0) log.debug("read data product") dp_obj = self.DPMS.read_data_product(dp_id) log.debug("find data products") self.assertIn(dp_id, [r._id for r in self.DPMS.find_data_products()]) log.debug("update data product") dp_obj.name = "brand new" self.DPMS.update_data_product(dp_obj) self.assertEqual("brand new", self.DPMS.read_data_product(dp_id).name) log.debug("activate/suspend persistence") self.assertFalse(self.DPMS.is_persisted(dp_id)) self.DPMS.activate_data_product_persistence(dp_id) self.addCleanup(self.DPMS.suspend_data_product_persistence, dp_id) # delete op will do this for us self.assertTrue(self.DPMS.is_persisted(dp_id)) log.debug("check error on checking persistence of nonexistent stream") #with self.assertRaises(NotFound): if True: self.DPMS.is_persisted(self.RR2.create(any_old(RT.DataProduct))) #log.debug("get extension") #self.DPMS.get_data_product_extension(dp_id) log.debug("getting last update") lastupdate = self.DPMS.get_last_update(dp_id) self.assertEqual({}, lastupdate) # should be no updates to a new data product log.debug("prepare resource support") support = self.DPMS.prepare_data_product_support(dp_id) self.assertIsNotNone(support) log.debug("delete data product") self.DPMS.delete_data_product(dp_id) log.debug("try to suspend again") self.DPMS.suspend_data_product_persistence(dp_id) # try basic create log.debug("create without a dataset") dp_id2 = self.DPMS.create_data_product_(any_old(RT.DataProduct)) self.assertIsNotNone(dp_id2) log.debug("activate product %s", dp_id2) self.assertRaises(BadRequest, self.DPMS.activate_data_product_persistence, dp_id2) #self.assertNotEqual(0, len(self.RR2.find_dataset_ids_of_data_product_using_has_dataset(dp_id2))) log.debug("force delete data product") self.DPMS.force_delete_data_product(dp_id2) log.debug("test complete")
class TestDataProductVersions(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.client = DataProductManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #@unittest.skip('not working') @patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}}) def test_createDataProductVersionSimple(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='test', parameter_dictionary_id=pdict_id) # test creating a new data product which will also create the initial/default version log.debug('Creating new data product with a stream definition') # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) dp_id = self.client.create_data_product(dp_obj, ctd_stream_def_id) log.debug('new dp_id = %s', str(dp_id)) dpc_id = self.client.create_data_product_collection(data_product_id=dp_id, collection_name='firstCollection', collection_description='collection desc') #test that the links exist version_ids, _ = self.rrclient.find_objects(subject=dpc_id, predicate=PRED.hasVersion, id_only=True) log.debug('version_ids = %s', str(version_ids)) self.assertTrue(version_ids, 'Failed to connect the data product to the version collection.') self.assertTrue(version_ids[0] == dp_id, 'Failed to connect the data product to the version collection.') # test creating a subsequent data product version which will update the data product pointers dp2_obj = IonObject(RT.DataProduct, name='DP2', description='a second dp', temporal_domain = tdom, spatial_domain = sdom) dp2_id = self.client.create_data_product(dp2_obj, ctd_stream_def_id) log.debug('second dp_id = %s', dp2_id) log.debug("adding product %s to collection %s", dp2_id, dpc_id) self.client.add_data_product_version_to_collection(data_product_id=dp2_id, data_product_collection_id=dpc_id, version_name="second version", version_description="a second version created" ) #test that the links exist version_ids = self.RR2.find_data_product_ids_of_data_product_collection_using_has_version(dpc_id) self.assertEqual(2, len(version_ids)) recent_version_id = self.client.get_current_version(dpc_id) self.assertEquals(recent_version_id, dp2_id) base_version_id = self.client.get_base_version(dpc_id) self.assertEquals(base_version_id, dp_id) #--------------------------------------------------------------------------------------------- # Now check that we can subscribe to the stream for the data product version #--------------------------------------------------------------------------------------------- # Activating the data products contained in the versions held by the data product collection data_product_collection_obj = self.rrclient.read(dpc_id) version_list = data_product_collection_obj.version_list self.assertEquals(len(version_list), 2) for version in version_list: data_product_id = version.data_product_id self.client.activate_data_product_persistence(data_product_id) streams = self.RR2.find_streams_of_data_product_using_has_stream(data_product_id) self.assertNotEqual(0, len(streams)) for stream in streams: self.assertTrue(stream.persisted) log.debug("This satisfies L4-CI-DM-RQ-053: 'The dynamic data distribution services shall support multiple " "versions of a given data topic.' This is true because we have shown above that we can persist the " "streams associated to the data product versions, and therefore we can subscribe to them. In " "test_oms_launch2.py, we have tested that activated data products like the ones we have here can be " "used by data processes to gather streaming data and it all works together correctly. This therefore " "completes the demonstration of req L4-CI-DM-RQ-053.") #--------------------------------------------------------------------------------------------- # Now delete all the created stuff and look for possible problems in doing so #--------------------------------------------------------------------------------------------- log.debug("deleting all data productgs") self.client.delete_data_product(dp_id) self.client.delete_data_product(dp2_id) self.client.delete_data_product_collection(dpc_id) self.client.force_delete_data_product_collection(dpc_id) # now try to get the deleted dp object self.assertRaises(NotFound, self.client.read_data_product_collection, dpc_id)
class TestCTDTransformsIntegration(IonIntegrationTestCase): pdict_id = None def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataset_management = self.datasetclient def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_instrument_model(self): instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) return instModel_id def _create_instrument_agent(self, instModel_id): raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) return instAgent_id def _create_instrument_device(self, instModel_id): instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) return instDevice_id def _create_instrument_agent_instance(self, instAgent_id, instDevice_id): port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) return instAgentInstance_id def _create_param_dicts(self): tdom, sdom = time_series_domain() self.sdom = sdom.dump() self.tdom = tdom.dump() self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) def _create_input_data_products( self, ctd_stream_def_id, instDevice_id, ): dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=self.tdom, spatial_domain=self.sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product( dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_parsed_data_product) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) #--------------------------------------------------------------------------- # Create CTD Raw as the second data product #--------------------------------------------------------------------------- if not self.pdict_id: self._create_param_dicts() raw_stream_def_id = self.pubsubclient.create_stream_definition( name='SBE37_RAW', parameter_dictionary_id=self.pdict_id) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=self.tdom, spatial_domain=self.sdom) ctd_raw_data_product = self.dataproductclient.create_data_product( dp_obj, raw_stream_def_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_raw_data_product) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids return ctd_parsed_data_product def _create_data_process_definitions(self): #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') self.ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L1 Conductivity: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') self.ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L1 Pressure: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') self.ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L1 Temperature: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') self.ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L2 Salinity: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') self.ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L2 Density: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') self.ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) return self.ctd_L0_all_dprocdef_id, self.ctd_L1_conductivity_dprocdef_id,\ self.ctd_L1_pressure_dprocdef_id,self.ctd_L1_temperature_dprocdef_id, \ self.ctd_L2_salinity_dprocdef_id, self.ctd_L2_density_dprocdef_id def _create_stream_definitions(self): if not self.pdict_id: self._create_param_dicts() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name='L0_Conductivity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, self.ctd_L0_all_dprocdef_id, binding='conductivity') outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name='L0_Pressure', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, self.ctd_L0_all_dprocdef_id, binding='pressure') outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name='L0_Temperature', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, self.ctd_L0_all_dprocdef_id, binding='temperature') return outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id def _create_l0_output_data_products(self, outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id): out_data_prods = [] ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) out_data_prods.append(self.ctd_l0_conductivity_output_dp_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l0_conductivity_output_dp_id) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) out_data_prods.append(self.ctd_l0_pressure_output_dp_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l0_pressure_output_dp_id) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) out_data_prods.append(self.ctd_l0_temperature_output_dp_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l0_temperature_output_dp_id) return out_data_prods def _create_l1_out_data_products(self): ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_conductivity_output_dp_obj, self.outgoing_stream_l1_conductivity_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l1_conductivity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_conductivity', stream_ids[0]) self.loggerpids.append(pid) ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_pressure_output_dp_obj, self.outgoing_stream_l1_pressure_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l1_pressure_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_pressure', stream_ids[0]) self.loggerpids.append(pid) ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_temperature_output_dp_obj, self.outgoing_stream_l1_temperature_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l1_temperature_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_temperature', stream_ids[0]) self.loggerpids.append(pid) def _create_l2_out_data_products(self): #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- if not self.pdict_id: self._create_param_dicts() outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition( name='L2_salinity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l2_salinity_id, self.ctd_L2_salinity_dprocdef_id, binding='salinity') outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition( name='L2_Density', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l2_density_id, self.ctd_L2_density_dprocdef_id, binding='density') ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product( ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l2_salinity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_salinity', stream_ids[0]) self.loggerpids.append(pid) ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product( ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l2_density_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l2_density_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_density', stream_ids[0]) self.loggerpids.append(pid) @unittest.skip('This test errors on coi-nightly, may be OBE.') def test_createTransformsThenActivateInstrument(self): self.loggerpids = [] #------------------------------------------------------------------------------------- # Create InstrumentModel #------------------------------------------------------------------------------------- instModel_id = self._create_instrument_model() #------------------------------------------------------------------------------------- # Create InstrumentAgent #------------------------------------------------------------------------------------- instAgent_id = self._create_instrument_agent(instModel_id) #------------------------------------------------------------------------------------- # Create InstrumentDevice #------------------------------------------------------------------------------------- instDevice_id = self._create_instrument_device(instModel_id) #------------------------------------------------------------------------------------- # Create Instrument Agent Instance #------------------------------------------------------------------------------------- instAgentInstance_id = self._create_instrument_agent_instance( instAgent_id, instDevice_id) #------------------------------------------------------------------------------------- # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------- self._create_param_dicts() ctd_stream_def_id = self.pubsubclient.create_stream_definition( name='SBE37_CDM', parameter_dictionary_id=self.pdict_id) #------------------------------------------------------------------------------------- # Create two data products #------------------------------------------------------------------------------------- ctd_parsed_data_product = self._create_input_data_products( ctd_stream_def_id, instDevice_id) #------------------------------------------------------------------------------------- # Create data process definitions #------------------------------------------------------------------------------------- self._create_data_process_definitions() #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------------------------------------------------------------- outgoing_stream_l0_conductivity_id, \ outgoing_stream_l0_pressure_id, \ outgoing_stream_l0_temperature_id = self._create_stream_definitions() self.out_prod_ids = self._create_l0_output_data_products( outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id) self.outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition( name='L1_conductivity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.outgoing_stream_l1_conductivity_id, self.ctd_L1_conductivity_dprocdef_id, binding='conductivity') self.outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition( name='L1_Pressure', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.outgoing_stream_l1_pressure_id, self.ctd_L1_pressure_dprocdef_id, binding='pressure') self.outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition( name='L1_Temperature', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.outgoing_stream_l1_temperature_id, self.ctd_L1_temperature_dprocdef_id, binding='temperature') self._create_l1_out_data_products() self._create_l2_out_data_products() #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------------------------------------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id=self.ctd_L0_all_dprocdef_id, in_data_product_ids=[ctd_parsed_data_product], out_data_product_ids=self.out_prod_ids) self.dataprocessclient.activate_data_process( ctd_l0_all_data_process_id) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) extended_process = self.dataprocessclient.get_data_process_extension( ctd_l0_all_data_process_id) self.assertEquals(extended_process.computed.operational_state.status, ComputedValueAvailability.NOTAVAILABLE) self.assertEquals(data_process.message_controllable, True) #------------------------------------------------------------------------------------- # L1 Conductivity: Create the data process #------------------------------------------------------------------------------------- l1_conductivity_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L1_conductivity_dprocdef_id, [self.ctd_l0_conductivity_output_dp_id], [self.ctd_l1_conductivity_output_dp_id]) self.dataprocessclient.activate_data_process( l1_conductivity_data_process_id) data_process = self.rrclient.read(l1_conductivity_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l1_conductivity_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L1 Pressure: Create the data process #------------------------------------------------------------------------------------- l1_pressure_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L1_pressure_dprocdef_id, [self.ctd_l0_pressure_output_dp_id], [self.ctd_l1_pressure_output_dp_id]) self.dataprocessclient.activate_data_process( l1_pressure_data_process_id) data_process = self.rrclient.read(l1_pressure_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l1_pressure_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L1 Temperature: Create the data process #------------------------------------------------------------------------------------- l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L1_temperature_dprocdef_id, [self.ctd_l0_temperature_output_dp_id], [self.ctd_l1_temperature_output_dp_id]) self.dataprocessclient.activate_data_process( l1_temperature_all_data_process_id) data_process = self.rrclient.read(l1_temperature_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l1_temperature_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L2 Salinity: Create the data process #------------------------------------------------------------------------------------- l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_salinity_output_dp_id]) self.dataprocessclient.activate_data_process( l2_salinity_all_data_process_id) data_process = self.rrclient.read(l2_salinity_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l2_salinity_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L2 Density: Create the data process #------------------------------------------------------------------------------------- l2_density_all_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_density_output_dp_id]) self.dataprocessclient.activate_data_process( l2_density_all_data_process_id) data_process = self.rrclient.read(l2_density_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l2_density_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------------------------------------------------------------- self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) # Wait for instrument agent to spawn gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (15), "The instrument agent instance did not spawn in 15 seconds") # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) #------------------------------------------------------------------------------------- # Streaming #------------------------------------------------------------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) #todo ResourceAgentClient no longer has method set_param # # Make sure the sampling rate and transmission are sane. # params = { # SBE37Parameter.NAVG : 1, # SBE37Parameter.INTERVAL : 5, # SBE37Parameter.TXREALTIME : True # } # self._ia_client.set_param(params) #todo There is no ResourceAgentEvent attribute for go_streaming... so what should be the command for it? cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # This gevent sleep is there to test the autosample time, which will show something different from default # only if the instrument runs for over a minute gevent.sleep(90) extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) autosample_string = extended_instrument.computed.uptime.value autosampling_time = int(autosample_string.split()[4]) self.assertTrue(autosampling_time > 0) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) #todo There is no ResourceAgentEvent attribute for go_observatory... so what should be the command for it? # log.debug("test_activateInstrumentStream: calling go_observatory") # cmd = AgentCommand(command='go_observatory') # reply = self._ia_client.execute_agent(cmd) # cmd = AgentCommand(command='get_current_state') # retval = self._ia_client.execute_agent(cmd) # state = retval.result # log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) #------------------------------------------------------------------------------------------------- # Cleanup processes #------------------------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) #-------------------------------------------------------------------------------- # Cleanup data products #-------------------------------------------------------------------------------- dp_ids, _ = self.rrclient.find_resources(restype=RT.DataProduct, id_only=True) for dp_id in dp_ids: self.dataproductclient.delete_data_product(dp_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain = tdom, spatial_domain = sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Lifecycle self.assertEquals(len(extended_instrument.lcstate_transitions), 7) self.assertEquals(set(extended_instrument.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce'])) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension(platform_device_id) self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) self.assertEquals(extended_platform.platform_agent._id, platform_agent_id) self.assertEquals(len(extended_platform.lcstate_transitions), 7) self.assertEquals(set(extended_platform.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce'])) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(StatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": {"favorite_color": "attr desc goes here"} })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) print "Saved config:" print snap_obj.content #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config(self): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC pconfig_builder = PlatformAgentConfigurationBuilder(clients) iconfig_builder = InstrumentAgentConfigurationBuilder(clients) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_obj = any_old(RT.Org) org_id = self.RR2.create(org_obj) inst_startup_config = {'startup': 'config'} generic_alerts_config = {'lvl1': {'lvl2': 'lvl3val'}} required_config_keys = [ 'org_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'aparam_alert_config', 'children'] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.name, config['org_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertEqual({'resource_id': device_id}, config['agent']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('aparam_alert_config', config) self.assertEqual(generic_alerts_config, config['aparam_alert_config']) self.assertIn('stream_config', config) for key in ['children']: self.assertEqual({}, config[key]) def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.name, config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('aparam_alert_config', config) self.assertEqual(generic_alerts_config, config['aparam_alert_config']) self.assertIn('stream_config', config) self.assertIn('driver_config', config) self.assertIn('foo', config['driver_config']) self.assertEqual('bar', config['driver_config']['foo']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) if None is inst_device_id: for key in ['children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.name, config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('aparam_alert_config', config) self.assertEqual(generic_alerts_config, config['aparam_alert_config']) self.assertIn('stream_config', config) for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': {'foo': 'bar'}, 'alerts': generic_alerts_config}) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config, 'alerts': generic_alerts_config}) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug("Testing that preparing a launcher without agent instance raises an error") self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False) log.debug("Making the structure for a platform agent, which will be the child") platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure() platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj) child_config = pconfig_builder.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug("Making the structure for a platform agent, which will be the parent") platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure() platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id) child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure() instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id) log.debug("Testing instrument config") iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj) instrument_config = iconfig_builder.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) full_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ datastore_name = CACHE_DATASTORE_NAME self.db = self.container.datastore_manager.get_datastore(datastore_name) self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.process_dispatcher.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) IngestionManagementIntTest.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Generic time-series data domain creation tdom, sdom = time_series_domain() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id2) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description,'the very first dp') #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) self.assertEqual(dp_id, extended_product._id) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual(0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product(dp_id) # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) # self.assertTrue(len(events) > 0) def test_data_product_stream_def(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id) self.assertEquals(ctd_stream_def_id, stream_def_id) def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) self.get_datastore(dataset_ids[0]) # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_ids[0]) self.assertIsInstance(replay_data, Granule) log.debug("The data retriever was able to replay the dataset that was attached to the data product " "we wanted to be persisted. Therefore the data product was indeed persisted with " "otherwise we could not have retrieved its dataset using the data retriever. Therefore " "this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'") data_product_object = self.rrclient.read(dp_id) self.assertEquals(data_product_object.name,'DP1') self.assertEquals(data_product_object.description,'some new dp') log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. " " Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the " "resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name, data_product_object.description)) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.rrclient.read(dp_id)
class TestActivateRSNVel3DInstrument(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateRSNVel3DInstrument, self).setUp() config = DotDict() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id, configuration=configuration) return pid @attr('LOCOINT') @unittest.skip('under construction') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 180}}}) def test_activate_rsn_vel3d(self): log.info("--------------------------------------------------------------------------------------------------------") # load_parameter_scenarios self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="BETA", path="master", categories="ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition", clearcols="owner_id,org_ids", assets="res/preload/r2_ioc/ooi_assets", parseooi="True", )) self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='Vel3DMModel', description="Vel3DMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug( 'test_activate_rsn_vel3d new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw' ) vel3d_b_sample = StreamConfiguration(stream_name='vel3d_b_sample', parameter_dictionary_name='vel3d_b_sample') vel3d_b_engineering = StreamConfiguration(stream_name='vel3d_b_engineering', parameter_dictionary_name='vel3d_b_engineering') RSN_VEL3D_01 = { 'DEV_ADDR' : "10.180.80.6", 'DEV_PORT' : 2101, 'DATA_PORT' : 1026, 'CMD_PORT' : 1025, 'PA_BINARY' : "port_agent" } # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='Vel3DAgent', description="Vel3DAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg", stream_configurations = [raw_config, vel3d_b_sample, vel3d_b_engineering]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('test_activate_rsn_vel3d new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activate_rsn_vel3d: Create instrument resource to represent the Vel3D ') instDevice_obj = IonObject(RT.InstrumentDevice, name='Vel3DDevice', description="Vel3DDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activate_rsn_vel3d: new InstrumentDevice id = %s " , instDevice_id) port_agent_config = { 'device_addr': '10.180.80.6', 'device_port': 2101, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': 1025, 'data_port': 1026, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='Vel3DAgentInstance', description="Vel3DAgentInstance", port_agent_config = port_agent_config, alerts= []) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_sample', id_only=True) parsed_sample_stream_def_id = self.pubsubcli.create_stream_definition(name='vel3d_b_sample', parameter_dictionary_id=parsed_sample_pdict_id) parsed_eng_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_engineering', id_only=True) parsed_eng_stream_def_id = self.pubsubcli.create_stream_definition(name='vel3d_b_engineering', parameter_dictionary_id=parsed_eng_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='vel3d_b_sample', description='vel3d_b_sample', temporal_domain = tdom, spatial_domain = sdom) sample_data_product_id = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_sample_stream_def_id) log.debug( 'new dp_id = %s' , sample_data_product_id) self.dpclient.activate_data_product_persistence(data_product_id=sample_data_product_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=sample_data_product_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasStream, None, True) log.debug('sample_data_product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for sample_data_product = %s' , dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('vel3d_b_sample', stream_ids[0] ) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='vel3d_b_engineering', description='vel3d_b_engineering', temporal_domain = tdom, spatial_domain = sdom) eng_data_product_id = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_eng_stream_def_id) log.debug( 'new dp_id = %s' , eng_data_product_id) self.dpclient.activate_data_product_persistence(data_product_id=eng_data_product_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=eng_data_product_id) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('test_activate_rsn_vel3d Data product streams2 = %s' , str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('test_activate_rsn_vel3d Data set for data_product_id2 = %s' , dataset_ids[0]) self.raw_dataset = dataset_ids[0] def start_instrument_agent(): self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) #log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) def check_state(label, desired_state): actual_state = self._ia_client.get_agent_state() log.debug("%s instrument agent is in state '%s'", label, actual_state) self.assertEqual(desired_state, actual_state) log.debug("test_activate_rsn_vel3d: got ia client %s" , str(self._ia_client)) check_state("just-spawned", ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: initialize %s" , str(retval)) check_state("initialized", ResourceAgentState.INACTIVE) log.debug("test_activate_rsn_vel3d Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: return value from go_active %s" , str(reply)) check_state("activated", ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("current state after sending go_active command %s" , str(state)) # cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) check_state("commanded", ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("current state after sending run command %s" , str(state)) # cmd = AgentCommand(command=ProtocolEvent.START_AUTOSAMPLE) # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # gevent.sleep(5) # # cmd = AgentCommand(command=ProtocolEvent.STOP_AUTOSAMPLE) # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) # retval = self._ia_client.execute_agent(cmd) # state = retval.result # log.debug("current state after sending STOP_AUTOSAMPLE command %s" , str(state)) # # cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.STOPPED, state) # # cmd = AgentCommand(command=ResourceAgentEvent.RESUME) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.IDLE, state) # # cmd = AgentCommand(command=ResourceAgentEvent.RUN) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) log.debug( "test_activate_rsn_vel3d: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: return from reset %s" , str(reply)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data_raw, Granule) rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw) log.debug("RDT raw: %s", str(rdt_raw.pretty_print()) ) self.assertIn('raw', rdt_raw) raw_vals = rdt_raw['raw'] #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(sample_data_product_id) self.dpclient.delete_data_product(eng_data_product_id) self.dpclient.delete_data_product(data_product_id2)
class TestDataProductManagementServiceCoverage(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() log.debug("Start rel from url") self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.DPMS = DataProductManagementServiceClient() self.RR = ResourceRegistryServiceClient() self.RR2 = EnhancedResourceRegistryClient(self.RR) self.DAMS = DataAcquisitionManagementServiceClient() self.PSMS = PubsubManagementServiceClient() self.ingestclient = IngestionManagementServiceClient() self.PD = ProcessDispatcherServiceClient() self.DSMS = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ log.debug("get datastore") datastore_name = CACHE_DATASTORE_NAME self.db = self.container.datastore_manager.get_datastore( datastore_name) self.stream_def_id = self.PSMS.create_stream_definition( name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition( name='ingestion worker') ingestion_worker_definition.executable = { 'module': 'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class': 'ScienceGranuleIngestionWorker' } process_definition_id = self.PD.create_process_definition( process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] self.addCleanup(self.cleaning_up) @staticmethod def clean_subscriptions(): ingestion_management = IngestionManagementServiceClient() pubsub = PubsubManagementServiceClient() rr = ResourceRegistryServiceClient() ingestion_config_ids = ingestion_management.list_ingestion_configurations( id_only=True) for ic in ingestion_config_ids: subscription_ids, assocs = rr.find_objects( subject=ic, predicate=PRED.hasSubscription, id_only=True) for subscription_id, assoc in zip(subscription_ids, assocs): rr.delete_association(assoc) try: pubsub.deactivate_subscription(subscription_id) except: log.exception("Unable to decativate subscription: %s", subscription_id) pubsub.delete_subscription(subscription_id) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.PD.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) TestDataProductManagementServiceCoverage.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def test_CRUD_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary = self.DSMS.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict') ctd_stream_def_id = self.PSMS.create_stream_definition( name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Generic time-series data domain creation tdom, sdom = time_series_domain() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain=tdom.dump(), spatial_domain=sdom.dump()) dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0 dp_obj.ooi_product_name = "PRODNAME" #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ log.debug("create dataset") dataset_id = self.RR2.create(any_old(RT.Dataset)) log.debug("dataset_id = %s", dataset_id) log.debug("create data product 1") dp_id = self.DPMS.create_data_product( data_product=dp_obj, stream_definition_id=ctd_stream_def_id, dataset_id=dataset_id) log.debug("dp_id = %s", dp_id) # Assert that the data product has an associated stream at this stage stream_ids, _ = self.RR.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertNotEquals(len(stream_ids), 0) log.debug("read data product") dp_obj = self.DPMS.read_data_product(dp_id) log.debug("find data products") self.assertIn(dp_id, [r._id for r in self.DPMS.find_data_products()]) log.debug("update data product") dp_obj.name = "brand new" self.DPMS.update_data_product(dp_obj) self.assertEqual("brand new", self.DPMS.read_data_product(dp_id).name) log.debug("activate/suspend persistence") self.assertFalse(self.DPMS.is_persisted(dp_id)) self.DPMS.activate_data_product_persistence(dp_id) self.addCleanup(self.DPMS.suspend_data_product_persistence, dp_id) # delete op will do this for us self.assertTrue(self.DPMS.is_persisted(dp_id)) log.debug("check error on checking persistence of nonexistent stream") #with self.assertRaises(NotFound): if True: self.DPMS.is_persisted(self.RR2.create(any_old(RT.DataProduct))) #log.debug("get extension") #self.DPMS.get_data_product_extension(dp_id) log.debug("getting last update") lastupdate = self.DPMS.get_last_update(dp_id) self.assertEqual( {}, lastupdate) # should be no updates to a new data product log.debug("prepare resource support") support = self.DPMS.prepare_data_product_support(dp_id) self.assertIsNotNone(support) log.debug("delete data product") self.DPMS.delete_data_product(dp_id) log.debug("try to suspend again") self.DPMS.suspend_data_product_persistence(dp_id) # try basic create log.debug("create without a dataset") dp_id2 = self.DPMS.create_data_product_(any_old(RT.DataProduct)) self.assertIsNotNone(dp_id2) log.debug("activate product %s", dp_id2) self.assertRaises(BadRequest, self.DPMS.activate_data_product_persistence, dp_id2) #self.assertNotEqual(0, len(self.RR2.find_dataset_ids_of_data_product_using_has_dataset(dp_id2))) log.debug("force delete data product") self.DPMS.force_delete_data_product(dp_id2) log.debug("test complete")
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateInstrumentIntegration, self).setUp() config = DotDict() config.bootstrap.use_es = True self._start_container() self.addCleanup(TestActivateInstrumentIntegration.es_cleanup) self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.usernotificationclient = UserNotificationServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() @staticmethod def es_cleanup(): es_host = CFG.get_safe('server.elasticsearch.host', 'localhost') es_port = CFG.get_safe('server.elasticsearch.port', '9200') es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10) indexes = STD_INDEXES.keys() indexes.append('%s_resources_index' % get_sys_name().lower()) indexes.append('%s_events_index' % get_sys_name().lower()) for index in indexes: IndexManagementService._es_call(es.river_couchdb_delete, index) IndexManagementService._es_call(es.index_delete, index) def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_notification(self, user_name='', instrument_id='', product_id=''): #-------------------------------------------------------------------------------------- # Make notification request objects #-------------------------------------------------------------------------------------- notification_request_1 = NotificationRequest( name='notification_1', origin=instrument_id, origin_type="instrument", event_type='ResourceLifecycleEvent') notification_request_2 = NotificationRequest( name='notification_2', origin=product_id, origin_type="data product", event_type='DetectionEvent') #-------------------------------------------------------------------------------------- # Create a user and get the user_id #-------------------------------------------------------------------------------------- user = UserInfo() user.name = user_name user.contact.email = '*****@*****.**' % user_name user_id, _ = self.rrclient.create(user) #-------------------------------------------------------------------------------------- # Create notification #-------------------------------------------------------------------------------------- self.usernotificationclient.create_notification( notification=notification_request_1, user_id=user_id) self.usernotificationclient.create_notification( notification=notification_request_2, user_id=user_id) log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id)) return user_id def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def _check_computed_attributes_of_extended_instrument( self, expected_instrument_device_id='', extended_instrument=None): # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance( extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance( extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance( extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.location_status_roll_up, ComputedIntValue) # the following assert will not work without elasticsearch. #self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) self.assertEqual( extended_instrument.computed.communications_status_roll_up.value, StatusType.STATUS_WARNING) self.assertEqual( extended_instrument.computed.data_status_roll_up.value, StatusType.STATUS_OK) self.assertEqual( extended_instrument.computed.power_status_roll_up.value, StatusType.STATUS_WARNING) # Verify the computed attribute for user notification requests self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value)) notifications = extended_instrument.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(notification.origin, expected_instrument_device_id) self.assertEqual(notification.origin_type, "instrument") self.assertEqual(notification.event_type, 'ResourceLifecycleEvent') def _check_computed_attributes_of_extended_product( self, expected_data_product_id='', extended_data_product=None): self.assertEqual(expected_data_product_id, extended_data_product._id) log.debug("extended_data_product.computed: %s", extended_data_product.computed) # Verify that computed attributes exist for the extended instrument self.assertIsInstance( extended_data_product.computed.product_download_size_estimated, ComputedIntValue) self.assertIsInstance( extended_data_product.computed.number_active_subscriptions, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue) self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue) self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue) self.assertIsInstance( extended_data_product.computed.user_notification_requests, ComputedListValue) self.assertIsInstance( extended_data_product.computed.active_user_subscriptions, ComputedListValue) self.assertIsInstance( extended_data_product.computed.past_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue) self.assertIsInstance( extended_data_product.computed.data_contents_updated, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue) # exact text here keeps changing to fit UI capabilities. keep assertion general... self.assertTrue('ok' in extended_data_product.computed.last_granule. value['quality_flag']) self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value)) notifications = extended_data_product.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(notification.origin, expected_data_product_id) self.assertEqual(notification.origin_type, "data product") self.assertEqual(notification.event_type, 'DetectionEvent') @attr('LOCOINT') @unittest.skipIf(not use_es, 'No ElasticSearch') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}}) def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', instModel_id) #Create stream alarms """ test_two_sided_interval Test interval alarm and alarm event publishing for a closed inteval. """ # kwargs = { # 'name' : 'test_sim_warning', # 'stream_name' : 'parsed', # 'value_id' : 'temp', # 'message' : 'Temperature is above test range of 5.0.', # 'type' : StreamAlarmType.WARNING, # 'upper_bound' : 5.0, # 'upper_rel_op' : '<' # } kwargs = { 'name': 'temperature_warning_interval', 'stream_name': 'parsed', 'value_id': 'temp', 'message': 'Temperature is below the normal range of 50.0 and above.', 'type': StreamAlarmType.WARNING, 'lower_bound': 50.0, 'lower_rel_op': '<' } # Create alarm object. alarm = {} alarm['type'] = 'IntervalAlarmDef' alarm['kwargs'] = kwargs raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5) parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5, alarms=[alarm]) # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri= "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg", stream_configurations=[raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ' ) instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) # setup notifications for the device and parsed data product user_id_1 = self._create_notification(user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1) #---------- Create notifications for another user and verify that we see different computed subscriptions for the two users --------- user_id_2 = self._create_notification(user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('Data product streams2 = %s', str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id2 = %s', dataset_ids[0]) self.raw_dataset = dataset_ids[0] #elastic search debug es_indexes, _ = self.container.resource_registry.find_resources( restype='ElasticSearchIndex') log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes]) log.debug('Bootstrap %s', CFG.bootstrap.use_es) def start_instrument_agent(): self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #setup a subscriber to alarm events from the device self._events_received = [] self._event_count = 0 self._samples_out_of_range = 0 self._samples_complete = False self._async_sample_result = AsyncResult() def consume_event(*args, **kwargs): log.debug( 'TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) self._event_count = len(self._events_received) self._async_sample_result.set() self._event_subscriber = EventSubscriber( event_type= 'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent callback=consume_event, origin=instDevice_id) self._event_subscriber.start() #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) def stop_subscriber(): self._event_subscriber.stop() self._event_subscriber = None self.addCleanup(stop_subscriber) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) gate = ProcessStateGate(self.processdispatchclient.read_process, inst_agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % inst_agent_instance_obj.agent_process_id) log.debug('Instrument agent instance obj: = %s', str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient( instDevice_id, to_name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s", str(retval)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) for i in xrange(10): retval = self._ia_client.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from sample %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) self._samples_complete = True #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt.pretty_print())) temp_vals = rdt['temp'] self.assertEquals(len(temp_vals), 10) log.debug("test_activateInstrumentSample: all temp_vals: %s", temp_vals) #out_of_range_temp_vals = [i for i in temp_vals if i > 5] out_of_range_temp_vals = [i for i in temp_vals if i < 50.0] log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s", out_of_range_temp_vals) self._samples_out_of_range = len(out_of_range_temp_vals) # if no bad values were produced, then do not wait for an event if self._samples_out_of_range == 0: self._async_sample_result.set() log.debug("test_activateInstrumentSample: _events_received: %s", self._events_received) log.debug("test_activateInstrumentSample: _event_count: %s", self._event_count) self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout) replay_data = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("RDT raw: %s", str(rdt.pretty_print())) raw_vals = rdt['raw'] self.assertEquals(len(raw_vals), 10) log.debug("l4-ci-sa-rq-138") """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ #-------------------------------------------------------------------------------- # Get the extended data product to see if it contains the granules #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id1, user_id=user_id_1) def poller(extended_product): return len(extended_product.computed.user_notification_requests. value) == 1 poll(poller, extended_product, timeout=30) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id1, extended_data_product=extended_product) #-------------------------------------------------------------------------------- #put some events into the eventsdb to test - this should set the comms and data status to WARNING #-------------------------------------------------------------------------------- t = get_ion_ts() self.event_publisher.publish_event(ts_created=t, event_type='DeviceStatusEvent', origin=instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values=[200]) self.event_publisher.publish_event( ts_created=t, event_type='DeviceCommsEvent', origin=instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds=20) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_1) self._check_computed_attributes_of_extended_instrument( expected_instrument_device_id=instDevice_id, extended_instrument=extended_instrument) #-------------------------------------------------------------------------------- # For the second user, check the extended data product and the extended intrument #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id2, user_id=user_id_2) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id2, extended_data_product=extended_product) #---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING --------- t = get_ion_ts() self.event_publisher.publish_event(ts_created=t, event_type='DeviceStatusEvent', origin=instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values=[200]) self.event_publisher.publish_event( ts_created=t, event_type='DeviceCommsEvent', origin=instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds=20) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_2) self._check_computed_attributes_of_extended_instrument( expected_instrument_device_id=instDevice_id, extended_instrument=extended_instrument) #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(data_product_id1) self.dpclient.delete_data_product(data_product_id2)
class TestDataProductProvenance(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.dpmsclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() # create missing data process definition dpd_obj = IonObject(RT.DataProcessDefinition, name=LOGICAL_TRANSFORM_DEFINITION_NAME, description="normally in preload", module='ion.processes.data.transforms.logical_transform', class_name='logical_transform') self.dataprocessclient.create_data_process_definition(dpd_obj) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dataprocessclient.deactivate_data_process(proc_id) self.dataprocessclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip('not ready') def test_get_provenance(self): #create a deployment with metadata and an initial site and device instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "") log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id)) # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id)) self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id) # Create InstrumentAgent parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [parsed_config] ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) #------------------------------- # Create CTD Parsed data product #------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) #------------------------------- # create a data product for the site to pass the OMS check.... we need to remove this check #------------------------------- dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id) self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id) #------------------------------- # Deploy instrument device to instrument site #------------------------------- deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id) self.imsclient.deploy_instrument_device(instDevice_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) self.omsclient.activate_deployment(deployment_id) inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False) log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) ) #------------------------------- # Create the agent instance #------------------------------- port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} log.debug("TestDataProductProvenance: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id log.debug("TestDataProductProvenance: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id log.debug("TestDataProductProvenance: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' ) outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' ) outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id) log.debug("TestDataProductProvenance: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id) log.debug("TestDataProductProvenance: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' ) log.debug("TestDataProductProvenance: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) log.debug("TestDataProductProvenance: create output data product L2 Density") # ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, # name='L2_Density', # description='transform output pressure', # temporal_domain = tdom, # spatial_domain = sdom) # # ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, # outgoing_stream_l2_density_id, # parameter_dictionary) contactInfo = ContactInformation() contactInfo.individual_names_given = "Bill" contactInfo.individual_name_family = "Smith" contactInfo.street_address = "111 First St" contactInfo.city = "San Diego" contactInfo.email = "*****@*****.**" contactInfo.phones = ["858-555-6666"] contactInfo.country = "USA" contactInfo.postal_code = "92123" ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', contacts = [contactInfo], iso_topic_category = "my_iso_topic_category_here", quality_control_level = "1", temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) #activate only this data process just for coverage self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) log.debug("TestDataProductProvenance: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'conductivity':ctd_l1_conductivity_output_dp_id}) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'pressure':ctd_l1_pressure_output_dp_id}) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'temperature':ctd_l1_temperature_output_dp_id}) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'salinity':ctd_l2_salinity_output_dp_id}) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_Density data_process start") try: l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'density':ctd_l2_density_output_dp_id}) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. # self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) # print 'activate_instrument: got ia client %s', self._ia_client # log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id) self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id) self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id) self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the provenance info for the ctd density data product #------------------------------- provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id) log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict)) #validate that products are represented self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)]) density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id]) #------------------------------- # Retrieve the extended resource for this data product #------------------------------- extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id) self.assertEqual(1, len(extended_product.data_processes) ) self.assertEqual(3, len(extended_product.process_input_data_products) ) # log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list)) # log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes)) # log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products)) # log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value)) #------------------------------- # Retrieve the extended resource for this data process #------------------------------- extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) # log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def)) # log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes)) # log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products)) self.assertEqual(1, len(extended_process_def.data_processes) ) self.assertEqual(3, len(extended_process_def.output_stream_definitions) ) self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process #------------------------------- # Request the xml report #------------------------------- results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id) #------------------------------- # Cleanup #------------------------------- self.dpmsclient.delete_data_product(ctd_parsed_data_product) self.dpmsclient.delete_data_product(log_data_product_id) self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
def test_createDataProduct(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' # Establish endpoint with container container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) #print 'got CC client' container_client.start_rel_from_url('res/deploy/r2sa.yml') print 'started services' # Now create client to DataProductManagementService client = DataProductManagementServiceClient(node=self.container.node) # test creating a new data product w/o a data producer print 'Creating new data product w/o a data producer' dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') try: dp_id = client.create_data_product(dp_obj) except BadRequest as ex: self.fail("failed to create new data product: %s" %ex) print 'new dp_id = ', dp_id # test creating a duplicate data product print 'Creating the same data product a second time (duplicate)' dp_obj.description = 'the first dp' try: dp_id = client.create_data_product(dp_obj) except BadRequest as ex: print ex else: self.fail("duplicate data product was created with the same name") """ # This is broken until the interceptor handles lists properly (w/o converting them to constants) # and DAMS works with pubsub_management.register_producer() correctly # test creating a new data product with a data producer print 'Creating new data product with a data producer' dp_obj = IonObject(RT.DataProduct, name='DP2', description='another new dp') data_producer_obj = IonObject(RT.DataProducer, name='DataProducer1', description='a new data producer') try: dp_id = client.create_data_product(dp_obj, data_producer_obj) except BadRequest as ex: self.fail("failed to create new data product") print 'new dp_id = ', dp_id """ # test reading a non-existent data product print 'reading non-existent data product' try: dp_obj = client.read_data_product('some_fake_id') except NotFound as ex: pass else: self.fail("non-existing data product was found during read: %s" %dp_obj) # update a data product (tests read also) print 'Updating data product' # first get the existing dp object try: dp_obj = client.read_data_product(dp_id) except NotFound as ex: self.fail("existing data product was not found during read") else: pass #print 'dp_obj = ', dp_obj # now tweak the object dp_obj.description = 'the very first dp' # now write the dp back to the registry try: update_result = client.update_data_product(dp_obj) except NotFound as ex: self.fail("existing data product was not found during update") except Conflict as ex: self.fail("revision conflict exception during data product update") else: self.assertTrue(update_result == True) # now get the dp back to see if it was updated try: dp_obj = client.read_data_product(dp_id) except NotFound as ex: self.fail("existing data product was not found during read") else: pass #print 'dp_obj = ', dp_obj self.assertTrue(dp_obj.description == 'the very first dp') # now 'delete' the data product print "deleting data product" try: delete_result = client.delete_data_product(dp_id) except NotFound as ex: self.fail("existing data product was not found during delete") self.assertTrue(delete_result == True) # now try to get the deleted dp object try: dp_obj = client.read_data_product(dp_id) except NotFound as ex: pass else: self.fail("deleted data product was found during read") # now try to delete the already deleted dp object print "deleting non-existing data product" try: delete_result = client.delete_data_product(dp_id) except NotFound as ex: pass else: self.fail("non-existing data product was found during delete")
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateInstrumentIntegration, self).setUp() config = DotDict() config.bootstrap.use_es = True self._start_container() self.addCleanup(TestActivateInstrumentIntegration.es_cleanup) self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.usernotificationclient = UserNotificationServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() @staticmethod def es_cleanup(): es_host = CFG.get_safe('server.elasticsearch.host', 'localhost') es_port = CFG.get_safe('server.elasticsearch.port', '9200') es = ep.ElasticSearch( host=es_host, port=es_port, timeout=10 ) indexes = STD_INDEXES.keys() indexes.append('%s_resources_index' % get_sys_name().lower()) indexes.append('%s_events_index' % get_sys_name().lower()) for index in indexes: IndexManagementService._es_call(es.river_couchdb_delete,index) IndexManagementService._es_call(es.index_delete,index) def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_notification(self, user_name = '', instrument_id='', product_id=''): #-------------------------------------------------------------------------------------- # Make notification request objects #-------------------------------------------------------------------------------------- notification_request_1 = NotificationRequest( name= 'notification_1', origin=instrument_id, origin_type="instrument", event_type='ResourceLifecycleEvent') notification_request_2 = NotificationRequest( name='notification_2', origin=product_id, origin_type="data product", event_type='DetectionEvent') #-------------------------------------------------------------------------------------- # Create a user and get the user_id #-------------------------------------------------------------------------------------- user = UserInfo() user.name = user_name user.contact.email = '*****@*****.**' % user_name user_id, _ = self.rrclient.create(user) #-------------------------------------------------------------------------------------- # Create notification #-------------------------------------------------------------------------------------- self.usernotificationclient.create_notification(notification=notification_request_1, user_id=user_id) self.usernotificationclient.create_notification(notification=notification_request_2, user_id=user_id) log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id) ) return user_id def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def _check_computed_attributes_of_extended_instrument(self, expected_instrument_device_id = '',extended_instrument = None): # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) # the following assert will not work without elasticsearch. #self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) self.assertEqual(extended_instrument.computed.communications_status_roll_up.value, StatusType.STATUS_WARNING) self.assertEqual(extended_instrument.computed.data_status_roll_up.value, StatusType.STATUS_OK) self.assertEqual(extended_instrument.computed.power_status_roll_up.value, StatusType.STATUS_WARNING) # Verify the computed attribute for user notification requests self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) notifications = extended_instrument.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(notification.origin, expected_instrument_device_id) self.assertEqual(notification.origin_type, "instrument") self.assertEqual(notification.event_type, 'ResourceLifecycleEvent') def _check_computed_attributes_of_extended_product(self, expected_data_product_id = '', extended_data_product = None): self.assertEqual(expected_data_product_id, extended_data_product._id) log.debug("extended_data_product.computed: %s", extended_data_product.computed) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_data_product.computed.product_download_size_estimated, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.number_active_subscriptions, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue) self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue) self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.user_notification_requests, ComputedListValue) self.assertIsInstance(extended_data_product.computed.active_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.past_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_contents_updated, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue) # exact text here keeps changing to fit UI capabilities. keep assertion general... self.assertTrue( 'ok' in extended_data_product.computed.last_granule.value['quality_flag'] ) self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value) ) notifications = extended_data_product.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(notification.origin, expected_data_product_id) self.assertEqual(notification.origin_type, "data product") self.assertEqual(notification.event_type, 'DetectionEvent') @attr('LOCOINT') @unittest.skipIf(not use_es, 'No ElasticSearch') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}}) def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) #Create stream alarms """ test_two_sided_interval Test interval alarm and alarm event publishing for a closed inteval. """ # kwargs = { # 'name' : 'test_sim_warning', # 'stream_name' : 'parsed', # 'value_id' : 'temp', # 'message' : 'Temperature is above test range of 5.0.', # 'type' : StreamAlarmType.WARNING, # 'upper_bound' : 5.0, # 'upper_rel_op' : '<' # } kwargs = { 'name' : 'temperature_warning_interval', 'stream_name' : 'parsed', 'value_id' : 'temp', 'message' : 'Temperature is below the normal range of 50.0 and above.', 'type' : StreamAlarmType.WARNING, 'lower_bound' : 50.0, 'lower_rel_op' : '<' } # Create alarm object. alarm = {} alarm['type'] = 'IntervalAlarmDef' alarm['kwargs'] = kwargs raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5, alarms=[alarm] ) # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg", stream_configurations = [raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) " , instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s' , data_product_id1) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s' , dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # setup notifications for the device and parsed data product user_id_1 = self._create_notification( user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1) #---------- Create notifications for another user and verify that we see different computed subscriptions for the two users --------- user_id_2 = self._create_notification( user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('Data product streams2 = %s' , str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id2 = %s' , dataset_ids[0]) self.raw_dataset = dataset_ids[0] #elastic search debug es_indexes, _ = self.container.resource_registry.find_resources(restype='ElasticSearchIndex') log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes]) log.debug('Bootstrap %s', CFG.bootstrap.use_es) def start_instrument_agent(): self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #setup a subscriber to alarm events from the device self._events_received= [] self._event_count = 0 self._samples_out_of_range = 0 self._samples_complete = False self._async_sample_result = AsyncResult() def consume_event(*args, **kwargs): log.debug('TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) self._event_count = len(self._events_received) self._async_sample_result.set() self._event_subscriber = EventSubscriber( event_type= 'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent callback=consume_event, origin=instDevice_id) self._event_subscriber.start() #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) def stop_subscriber(): self._event_subscriber.stop() self._event_subscriber = None self.addCleanup(stop_subscriber) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.processdispatchclient.read_process, inst_agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % inst_agent_instance_obj.agent_process_id) log.debug('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s" , str(self._ia_client)) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s" , str(retval)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s" , str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s" , str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s" , str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) for i in xrange(10): retval = self._ia_client.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from sample %s" , str(retval)) log.debug( "test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s" , str(reply)) self._samples_complete = True #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt.pretty_print()) ) temp_vals = rdt['temp'] self.assertEquals(len(temp_vals) , 10) log.debug("test_activateInstrumentSample: all temp_vals: %s", temp_vals ) #out_of_range_temp_vals = [i for i in temp_vals if i > 5] out_of_range_temp_vals = [i for i in temp_vals if i < 50.0] log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s", out_of_range_temp_vals ) self._samples_out_of_range = len(out_of_range_temp_vals) # if no bad values were produced, then do not wait for an event if self._samples_out_of_range == 0: self._async_sample_result.set() log.debug("test_activateInstrumentSample: _events_received: %s", self._events_received ) log.debug("test_activateInstrumentSample: _event_count: %s", self._event_count ) self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout) replay_data = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("RDT raw: %s", str(rdt.pretty_print()) ) raw_vals = rdt['raw'] self.assertEquals(len(raw_vals) , 10) log.debug("l4-ci-sa-rq-138") """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ #-------------------------------------------------------------------------------- # Get the extended data product to see if it contains the granules #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id1, user_id=user_id_1) def poller(extended_product): return len(extended_product.computed.user_notification_requests.value) == 1 poll(poller, extended_product, timeout=30) self._check_computed_attributes_of_extended_product( expected_data_product_id = data_product_id1, extended_data_product = extended_product) #-------------------------------------------------------------------------------- #put some events into the eventsdb to test - this should set the comms and data status to WARNING #-------------------------------------------------------------------------------- t = get_ion_ts() self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceStatusEvent', origin = instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values = [200] ) self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceCommsEvent', origin = instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds = 20 ) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_1) self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument) #-------------------------------------------------------------------------------- # For the second user, check the extended data product and the extended intrument #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id2, user_id=user_id_2) self._check_computed_attributes_of_extended_product(expected_data_product_id = data_product_id2, extended_data_product = extended_product) #---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING --------- t = get_ion_ts() self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceStatusEvent', origin = instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values = [200] ) self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceCommsEvent', origin = instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds = 20 ) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_2) self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument) #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(data_product_id1) self.dpclient.delete_data_product(data_product_id2)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient() self.rrclient = ResourceRegistryServiceClient() self.damsclient = DataAcquisitionManagementServiceClient() self.pubsubcli = PubsubManagementServiceClient() self.ingestclient = IngestionManagementServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() self.identcli = IdentityManagementServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.process_dispatcher.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) IngestionManagementIntTest.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore @attr('EXT') @attr('PREP') def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0 dp_obj.ooi_product_name = "PRODNAME" #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id) # Assert that the data product has an associated stream at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertNotEquals(len(stream_ids), 0) # Assert that the data product has an associated stream def at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True) self.assertNotEquals(len(stream_ids), 0) self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Created data product %s', dp_obj) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp') dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id2) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) group_names = self.dpsc_cli.get_data_product_group_list() self.assertIn("PRODNAME", group_names) #---------------------------------------------------------------------------------------- # Create users then notifications to this data product for each user #---------------------------------------------------------------------------------------- # user_1 user_1 = UserInfo() user_1.name = 'user_1' user_1.contact.email = '*****@*****.**' # user_2 user_2 = UserInfo() user_2.name = 'user_2' user_2.contact.email = '*****@*****.**' #user1 is a complete user self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) actor_id = self.identcli.create_actor_identity(actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identcli.register_user_credentials(actor_id, user_credentials_obj) user_id_1 = self.identcli.create_user_info(actor_id, user_1) user_id_2, _ = self.rrclient.create(user_2) delivery_config1a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) delivery_config1b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) notification_request_1 = NotificationRequest( name = "notification_1", origin=dp_id, origin_type="type_1", event_type=OT.ResourceLifecycleEvent, disabled_by_system = False, delivery_configurations=[delivery_config1a, delivery_config1b]) delivery_config2a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) delivery_config2b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) notification_request_2 = NotificationRequest( name = "notification_2", origin=dp_id, origin_type="type_2", disabled_by_system = False, event_type=OT.DetectionEvent, delivery_configurations=[delivery_config2a, delivery_config2b]) notification_request_1_id = self.unsc.create_notification(notification=notification_request_1, user_id=user_id_1) notification_request_2_id = self.unsc.create_notification(notification=notification_request_2, user_id=user_id_2) self.unsc.delete_notification(notification_request_1_id) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0 # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description,'the very first dp') self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Updated data product %s', dp_obj) #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) #validate that there is one active and one retired user notification for this data product self.assertEqual(1, len(extended_product.computed.active_user_subscriptions.value)) self.assertEqual(1, len(extended_product.computed.past_user_subscriptions.value)) self.assertEqual(dp_id, extended_product._id) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual(0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) def ion_object_encoder(obj): return obj.__dict__ #test prepare for create data_product_data = self.dpsc_cli.prepare_data_product_support() #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, "") self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0) self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 0) self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0) #test prepare for update data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id) #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, dp_id) self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual(len(data_product_data.associations['Dataset'].resources), 1) self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1) self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, dp_id) self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 1) self.assertEqual(data_product_data.associations['Dataset'].associated_resources[0].s, dp_id) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) # Assert that there are no associated streams leftover after deleting the data product stream_ids, assoc_ids = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertEquals(len(stream_ids), 0) self.assertEquals(len(assoc_ids), 0) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product(dp_id) # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) self.assertTrue(len(events) > 0) def test_data_product_stream_def(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id) self.assertEquals(ctd_stream_def_id, stream_def_id) def test_derived_data_product(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='ctd parsed', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id) dp = DataProduct(name='Instrument DP') dp_id = self.dpsc_cli.create_data_product(dp, stream_definition_id=ctd_stream_def_id) self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id) self.dpsc_cli.activate_data_product_persistence(dp_id) self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Make the derived data product simple_stream_def_id = self.pubsubcli.create_stream_definition(name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time','temp']) tempwat_dp = DataProduct(name='TEMPWAT', category=DataProductTypeEnum.DERIVED) tempwat_dp_id = self.dpsc_cli.create_data_product(tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id) self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id) # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) rdt['pressure'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id,route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True) tempwat_dataset_id = tempwat_dataset_ids[0] granule = self.data_retriever.retrieve(tempwat_dataset_id, delivery_format=simple_stream_def_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['time'], np.arange(20)) self.assertEquals(set(rdt.fields), set(['time','temp'])) def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Construct temporal and spatial Coordinate Reference System objects dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # Subscribe to persist events #------------------------------------------------------------------------------------------------ queue = gevent.queue.Queue() def info_event_received(message, headers): queue.put(message) es = EventSubscriber(event_type=OT.InformationContentStatusEvent, callback=info_event_received, origin=dp_id, auto_delete=True) es.start() self.addCleanup(es.stop) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id,route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_ids[0]) self.assertIsInstance(replay_data, Granule) log.debug("The data retriever was able to replay the dataset that was attached to the data product " "we wanted to be persisted. Therefore the data product was indeed persisted with " "otherwise we could not have retrieved its dataset using the data retriever. Therefore " "this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'") data_product_object = self.rrclient.read(dp_id) self.assertEquals(data_product_object.name,'DP1') self.assertEquals(data_product_object.description,'some new dp') log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. " " Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the " "resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name, data_product_object.description)) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) dataset_modified.clear() rdt['time'] = np.arange(20,40) publisher.publish(rdt.to_granule()) self.assertFalse(dataset_modified.wait(2)) self.dpsc_cli.activate_data_product_persistence(dp_id) dataset_modified.clear() publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_almost_equal(rdt['time'], np.arange(40)) dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True) self.assertEquals(len(dataset_ids), 1) self.dpsc_cli.suspend_data_product_persistence(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.rrclient.read(dp_id) info_event_counter = 0 runtime = 0 starttime = time.time() caught_events = [] #check that the four InfoStatusEvents were received while info_event_counter < 4 and runtime < 60 : a = queue.get(timeout=60) caught_events.append(a) info_event_counter += 1 runtime = time.time() - starttime self.assertEquals(info_event_counter, 4)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ definition = SBE37_CDM_stream_definition() datastore_name = CACHE_DATASTORE_NAME self.db = self.container.datastore_manager.get_datastore(datastore_name) self.stream_def_id = self.pubsubcli.create_stream_definition(container=definition) self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config) @unittest.skip('OBE') def test_get_last_update(self): craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) data_product_id = self.dpsc_cli.create_data_product(data_product=dp_obj, stream_definition_id=self.stream_def_id, parameter_dictionary=parameter_dictionary) stream_ids, garbage = self.rrclient.find_objects(data_product_id, PRED.hasStream, id_only=True) stream_id = stream_ids[0] fake_lu = LastUpdate() fake_lu_doc = self.db._ion_object_to_persistence_dict(fake_lu) self.db.create_doc(fake_lu_doc, object_id=stream_id) #------------------------------------------ # Now execute #------------------------------------------ res = self.dpsc_cli.get_last_update(data_product_id=data_product_id) self.assertTrue(isinstance(res[stream_id], LastUpdate), 'retrieving documents failed') def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ ctd_stream_def = ctd_stream_definition() ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def, name='Simulated CTD data') log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ log.debug('test_createDataProduct: Creating new data product w/o a stream definition (L4-CI-SA-RQ-308)') craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ log.debug("parameter dictionary: %s" % parameter_dictionary) dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id, parameter_dictionary= parameter_dictionary) dp_obj = self.dpsc_cli.read_data_product(dp_id) log.debug('new dp_id = %s' % dp_id) log.debug("test_createDataProduct: Data product info from registry %s (L4-CI-SA-RQ-308)", str(dp_obj)) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description,'the very first dp') # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) # now try to get the deleted dp object #todo: the RR should perhaps not return retired data products # dp_obj = self.dpsc_cli.read_data_product(dp_id) # now try to delete the already deleted dp object # log.debug( "deleting non-existing data product") # self.dpsc_cli.delete_data_product(dp_id) # Shut down container #container.stop() def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ ctd_stream_def = ctd_stream_definition() ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def, name='Simulated CTD data') log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ log.debug('test_createDataProduct: Creating new data product w/o a stream definition (L4-CI-SA-RQ-308)') craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ log.debug("parameter dictionary: %s" % parameter_dictionary) dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id, parameter_dictionary= parameter_dictionary) dp_obj = self.dpsc_cli.read_data_product(dp_id) log.debug('new dp_id = %s' % dp_id) log.debug("test_createDataProduct: Data product info from registry %s (L4-CI-SA-RQ-308)", str(dp_obj)) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id)
class TestAgentLaunchOps(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' unittest # suppress an pycharm inspector error if all unittest.skip references are commented out self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return def test_get_agent_client_noprocess(self): inst_device_id = self.RR2.create(any_old(RT.InstrumentDevice)) iap = ResourceAgentClient._get_agent_process_id(inst_device_id) # should be no running agent self.assertIsNone(iap) # should raise NotFound self.assertRaises(NotFound, ResourceAgentClient, inst_device_id) def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict' ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = AgentProcessStateGate(self.PDC.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) if "BAD_DATA" == instance_obj.driver_config["comms_config"]: print "Saved config:" print snap_obj.content self.fail("Saved config was not properly restored") self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config_hasDevice(self): def assign_fn(child_device_id, parent_device_id): self.RR2.create_association(parent_device_id, PRED.hasDevice, child_device_id) def find_fn(parent_device_id): ret, _ = self.RR.find_objects(subject=parent_device_id, predicate=PRED.hasDevice, id_only=True) return ret self.base_agent_instance_config(assign_fn, find_fn) log.info("END test_agent_instance_config_hasDevice") def test_agent_instance_config_hasNetworkParent(self): def assign_fn(child_device_id, parent_device_id): self.RR2.create_association(child_device_id, PRED.hasNetworkParent, parent_device_id) def find_fn(parent_device_id): ret, _ = self.RR.find_subjects(object=parent_device_id, predicate=PRED.hasNetworkParent, id_only=True) return ret self.base_agent_instance_config(assign_fn, find_fn) log.info("END test_agent_instance_config_hasNetworkParent") def base_agent_instance_config(self, assign_child_platform_to_parent_platform_fn, find_child_platform_ids_of_parent_platform_fn): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC config_builder = DotDict config_builder.i = None config_builder.p = None def refresh_pconfig_builder_hack(config_builder): """ ugly hack to get around "idempotent" RR2 caching remove after https://github.com/ooici/coi-services/pull/1190 """ config_builder.p = PlatformAgentConfigurationBuilder(clients) def refresh_iconfig_builder_hack(config_builder): """ ugly hack to get around "idempotent" RR2 caching remove after https://github.com/ooici/coi-services/pull/1190 """ config_builder.i = InstrumentAgentConfigurationBuilder(clients) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_obj = any_old(RT.Org) org_id = self.RR2.create(org_obj) inst_startup_config = {'startup': 'config'} generic_alerts_config = [ {'lvl2': 'lvl3val'} ] required_config_keys = [ 'org_governance_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'aparam_alerts_config', 'children'] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertEqual({'resource_id': device_id}, config['agent']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) for key in ['children']: self.assertEqual({}, config[key]) def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) self.assertIn('driver_config', config) self.assertIn('foo', config['driver_config']) self.assertIn('ports', config['driver_config']) self.assertEqual('bar', config['driver_config']['foo']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) if None is inst_device_id: for key in ['children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) if config['driver_config']['ports']: self.assertTrue( isinstance(config['driver_config']['ports'], dict) ) def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertIn('process_type', config['driver_config']) self.assertIn('ports', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) for key in ['startup_config']: self.assertEqual({}, config[key]) if config['driver_config']['ports']: self.assertTrue( isinstance(config['driver_config']['ports'], dict) ) self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(name='', agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': {'foo': 'bar'}, 'alerts': generic_alerts_config}) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict' ) platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.suspend_data_product_persistence, dp_id) #deployment creation site_obj = IonObject(RT.PlatformSite, name='sitePlatform') site_id = self.OMS.create_platform_site(platform_site=site_obj) # find current deployment using time constraints current_time = int( calendar.timegm(time.gmtime()) ) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) platform_port_obj= IonObject(OT.PlatformPort, reference_designator = 'GA01SUMO-FI003-09-CTDMO0999', port_type=PortTypeEnum.UPLINK, ip_address=0) deployment_obj = IonObject(RT.Deployment, name='TestPlatformDeployment_' + name, description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds], port_assignments={platform_device_id:platform_port_obj}) deploy_id = self.OMS.create_deployment(deployment=deployment_obj, site_id=site_id, device_id=platform_device_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config, 'alerts': generic_alerts_config}) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict' ) instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.suspend_data_product_persistence, dp_id) #deployment creation site_obj = IonObject(RT.InstrumentSite, name='siteInstrument') site_id = self.OMS.create_instrument_site(instrument_site =site_obj) # find current deployment using time constraints current_time = int( calendar.timegm(time.gmtime()) ) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) platform_port_obj= IonObject(OT.PlatformPort, reference_designator = 'GA01SUMO-FI003-08-CTDMO0888', port_type=PortTypeEnum.PAYLOAD, ip_address=0) deployment_obj = IonObject(RT.Deployment, name='TestDeployment for Cabled Instrument', description='some new deployment', context=IonObject(OT.CabledInstrumentDeploymentContext), constraint_list=[temporal_bounds], port_assignments={instrument_device_id:platform_port_obj}) deploy_id = self.OMS.create_deployment(deployment=deployment_obj, site_id=site_id, device_id=instrument_device_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device_with_has_agent_instance(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance_with_has_agent_definition(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug("Testing that preparing a launcher without agent instance raises an error") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated self.assertRaises(AssertionError, config_builder.p.prepare, will_launch=False) log.debug("Making the structure for a platform agent, which will be the child") platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure(name='child') platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object(platform_agent_instance_child_obj) child_config = config_builder.p.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug("Making the structure for a platform agent, which will be the parent") platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure(name='parent') platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = config_builder.p.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") assign_child_platform_to_parent_platform_fn(platform_device_child_id, platform_device_parent_id) child_device_ids = find_child_platform_ids_of_parent_platform_fn(platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = config_builder.p.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure() instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id) log.debug("Testing instrument config") refresh_iconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.i.set_agent_instance_object(instrument_agent_instance_obj) instrument_config = config_builder.i.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device_with_has_device(instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_platform_device_using_has_device(platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object(platform_agent_instance_parent_obj) full_config = config_builder.p.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id) #self.fail(parent_config) #plauncher.prepare(will_launch=False) log.info("END base_agent_instance_config")
class TestCTDTransformsIntegration(IonIntegrationTestCase): pdict_id = None def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataset_management = self.datasetclient def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration) return pid def _create_instrument_model(self): instModel_obj = IonObject( RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) instModel_id = self.imsclient.create_instrument_model(instModel_obj) return instModel_id def _create_instrument_agent(self, instModel_id): raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) return instAgent_id def _create_instrument_device(self, instModel_id): instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) return instDevice_id def _create_instrument_agent_instance(self, instAgent_id,instDevice_id): port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) return instAgentInstance_id def _create_param_dicts(self): tdom, sdom = time_series_domain() self.sdom = sdom.dump() self.tdom = tdom.dump() self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) def _create_input_data_products(self, ctd_stream_def_id, instDevice_id, ): dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = self.tdom, spatial_domain = self.sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) #--------------------------------------------------------------------------- # Create CTD Raw as the second data product #--------------------------------------------------------------------------- if not self.pdict_id: self._create_param_dicts() raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=self.pdict_id) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = self.tdom, spatial_domain = self.sdom) ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids return ctd_parsed_data_product def _create_data_process_definitions(self): #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') self.ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L1 Conductivity: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') self.ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L1 Pressure: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') self.ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L1 Temperature: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') self.ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L2 Salinity: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') self.ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L2 Density: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') self.ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) return self.ctd_L0_all_dprocdef_id, self.ctd_L1_conductivity_dprocdef_id,\ self.ctd_L1_pressure_dprocdef_id,self.ctd_L1_temperature_dprocdef_id, \ self.ctd_L2_salinity_dprocdef_id, self.ctd_L2_density_dprocdef_id def _create_stream_definitions(self): if not self.pdict_id: self._create_param_dicts() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, self.ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, self.ctd_L0_all_dprocdef_id, binding= 'pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, self.ctd_L0_all_dprocdef_id, binding='temperature' ) return outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id def _create_l0_output_data_products(self, outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id): out_data_prods = [] ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) out_data_prods.append(self.ctd_l0_conductivity_output_dp_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_conductivity_output_dp_id) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) out_data_prods.append(self.ctd_l0_pressure_output_dp_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_pressure_output_dp_id) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) out_data_prods.append(self.ctd_l0_temperature_output_dp_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_temperature_output_dp_id) return out_data_prods def _create_l1_out_data_products(self): ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj, self.outgoing_stream_l1_conductivity_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_conductivity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_conductivity', stream_ids[0] ) self.loggerpids.append(pid) ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj, self.outgoing_stream_l1_pressure_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_pressure_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_pressure', stream_ids[0] ) self.loggerpids.append(pid) ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj, self.outgoing_stream_l1_temperature_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_temperature_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_temperature', stream_ids[0] ) self.loggerpids.append(pid) def _create_l2_out_data_products(self): #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- if not self.pdict_id: self._create_param_dicts() outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, self.ctd_L2_salinity_dprocdef_id, binding='salinity' ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, self.ctd_L2_density_dprocdef_id, binding='density' ) ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_salinity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_salinity', stream_ids[0] ) self.loggerpids.append(pid) ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_density_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_density_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_density', stream_ids[0] ) self.loggerpids.append(pid) @unittest.skip('This test errors on coi-nightly, may be OBE.') def test_createTransformsThenActivateInstrument(self): self.loggerpids = [] #------------------------------------------------------------------------------------- # Create InstrumentModel #------------------------------------------------------------------------------------- instModel_id = self._create_instrument_model() #------------------------------------------------------------------------------------- # Create InstrumentAgent #------------------------------------------------------------------------------------- instAgent_id = self._create_instrument_agent(instModel_id) #------------------------------------------------------------------------------------- # Create InstrumentDevice #------------------------------------------------------------------------------------- instDevice_id = self._create_instrument_device(instModel_id) #------------------------------------------------------------------------------------- # Create Instrument Agent Instance #------------------------------------------------------------------------------------- instAgentInstance_id = self._create_instrument_agent_instance(instAgent_id,instDevice_id ) #------------------------------------------------------------------------------------- # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------- self._create_param_dicts() ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=self.pdict_id) #------------------------------------------------------------------------------------- # Create two data products #------------------------------------------------------------------------------------- ctd_parsed_data_product = self._create_input_data_products(ctd_stream_def_id,instDevice_id) #------------------------------------------------------------------------------------- # Create data process definitions #------------------------------------------------------------------------------------- self._create_data_process_definitions() #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------------------------------------------------------------- outgoing_stream_l0_conductivity_id, \ outgoing_stream_l0_pressure_id, \ outgoing_stream_l0_temperature_id = self._create_stream_definitions() self.out_prod_ids = self._create_l0_output_data_products(outgoing_stream_l0_conductivity_id,outgoing_stream_l0_pressure_id,outgoing_stream_l0_temperature_id) self.outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.outgoing_stream_l1_conductivity_id, self.ctd_L1_conductivity_dprocdef_id, binding='conductivity' ) self.outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.outgoing_stream_l1_pressure_id, self.ctd_L1_pressure_dprocdef_id, binding='pressure' ) self.outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.outgoing_stream_l1_temperature_id, self.ctd_L1_temperature_dprocdef_id, binding= 'temperature' ) self._create_l1_out_data_products() self._create_l2_out_data_products() #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------------------------------------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = self.ctd_L0_all_dprocdef_id, in_data_product_ids = [ctd_parsed_data_product], out_data_product_ids = self.out_prod_ids) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process,process_ids[0]) extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id) self.assertEquals(extended_process.computed.operational_state.status, ComputedValueAvailability.NOTAVAILABLE) self.assertEquals(data_process.message_controllable, True) #------------------------------------------------------------------------------------- # L1 Conductivity: Create the data process #------------------------------------------------------------------------------------- l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L1_conductivity_dprocdef_id, [self.ctd_l0_conductivity_output_dp_id], [self.ctd_l1_conductivity_output_dp_id]) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) data_process = self.rrclient.read(l1_conductivity_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l1_conductivity_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process,process_ids[0]) #------------------------------------------------------------------------------------- # L1 Pressure: Create the data process #------------------------------------------------------------------------------------- l1_pressure_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L1_pressure_dprocdef_id, [self.ctd_l0_pressure_output_dp_id], [self.ctd_l1_pressure_output_dp_id]) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) data_process = self.rrclient.read(l1_pressure_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l1_pressure_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L1 Temperature: Create the data process #------------------------------------------------------------------------------------- l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L1_temperature_dprocdef_id, [self.ctd_l0_temperature_output_dp_id], [self.ctd_l1_temperature_output_dp_id]) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) data_process = self.rrclient.read(l1_temperature_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l1_temperature_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L2 Salinity: Create the data process #------------------------------------------------------------------------------------- l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_salinity_output_dp_id]) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) data_process = self.rrclient.read(l2_salinity_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l2_salinity_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L2 Density: Create the data process #------------------------------------------------------------------------------------- l2_density_all_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_density_output_dp_id]) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) data_process = self.rrclient.read(l2_density_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l2_density_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------------------------------------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) # Wait for instrument agent to spawn gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(15), "The instrument agent instance did not spawn in 15 seconds") # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) #------------------------------------------------------------------------------------- # Streaming #------------------------------------------------------------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) #todo ResourceAgentClient no longer has method set_param # # Make sure the sampling rate and transmission are sane. # params = { # SBE37Parameter.NAVG : 1, # SBE37Parameter.INTERVAL : 5, # SBE37Parameter.TXREALTIME : True # } # self._ia_client.set_param(params) #todo There is no ResourceAgentEvent attribute for go_streaming... so what should be the command for it? cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # This gevent sleep is there to test the autosample time, which will show something different from default # only if the instrument runs for over a minute gevent.sleep(90) extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) autosample_string = extended_instrument.computed.uptime.value autosampling_time = int(autosample_string.split()[4]) self.assertTrue(autosampling_time > 0) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) #todo There is no ResourceAgentEvent attribute for go_observatory... so what should be the command for it? # log.debug("test_activateInstrumentStream: calling go_observatory") # cmd = AgentCommand(command='go_observatory') # reply = self._ia_client.execute_agent(cmd) # cmd = AgentCommand(command='get_current_state') # retval = self._ia_client.execute_agent(cmd) # state = retval.result # log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) #------------------------------------------------------------------------------------------------- # Cleanup processes #------------------------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) #-------------------------------------------------------------------------------- # Cleanup data products #-------------------------------------------------------------------------------- dp_ids, _ = self.rrclient.find_resources(restype=RT.DataProduct, id_only=True) for dp_id in dp_ids: self.dataproductclient.delete_data_product(dp_id)
class TestDataProductProvenance(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.dpmsclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dataprocessclient.deactivate_data_process(proc_id) self.dataprocessclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) def test_get_data_product_provenance_report(self): #Create a test device device_obj = Device(name='Device1', description='test instrument site') device_id, _ = self.rrclient.create(device_obj) self.addCleanup(self.rrclient.delete, device_id) #Create a test DataProduct data_product1_obj = DataProduct(name='DataProduct1', description='test data product 1') data_product1_id, _ = self.rrclient.create(data_product1_obj) self.addCleanup(self.rrclient.delete, data_product1_id) #Create a test DataProcess data_process_obj = DataProcess(name='DataProcess', description='test data process') data_process_id, _ = self.rrclient.create(data_process_obj) self.addCleanup(self.rrclient.delete, data_process_id) #Create a second test DataProduct data_product2_obj = DataProduct(name='DataProduct2', description='test data product 2') data_product2_id, _ = self.rrclient.create(data_product2_obj) self.addCleanup(self.rrclient.delete, data_product2_id) #Create a test DataProducer data_producer_obj = DataProducer(name='DataProducer', description='test data producer') data_producer_id, rev = self.rrclient.create(data_producer_obj) #Link the DataProcess to the second DataProduct manually assoc_id, _ = self.rrclient.create_association(subject=data_process_id, predicate=PRED.hasInputProduct, object=data_product2_id) self.addCleanup(self.rrclient.delete_association, assoc_id) # Register the instrument and process. This links the device and the data process # with their own producers self.damsclient.register_instrument(device_id) self.addCleanup(self.damsclient.unregister_instrument, device_id) self.damsclient.register_process(data_process_id) self.addCleanup(self.damsclient.unregister_process, data_process_id) #Manually link the first DataProduct with the test DataProducer assoc_id, _ = self.rrclient.create_association(subject=data_product1_id, predicate=PRED.hasDataProducer, object=data_producer_id) #Get the DataProducer linked to the DataProcess (created in register_process above) #Associate that with with DataProduct1's DataProducer data_process_producer_ids, _ = self.rrclient.find_objects(subject=data_process_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True) assoc_id, _ = self.rrclient.create_association(subject=data_process_producer_ids[0], predicate=PRED.hasParent, object=data_producer_id) self.addCleanup(self.rrclient.delete_association, assoc_id) #Get the DataProducer linked to the Device (created in register_instrument #Associate that with the DataProcess's DataProducer device_producer_ids, _ = self.rrclient.find_objects(subject=device_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True) assoc_id, _ = self.rrclient.create_association(subject=data_producer_id, predicate=PRED.hasParent, object=device_producer_ids[0]) #Create the links between the Device, DataProducts, DataProcess, and all DataProducers self.damsclient.assign_data_product(input_resource_id=device_id, data_product_id=data_product1_id) self.addCleanup(self.damsclient.unassign_data_product, device_id, data_product1_id) self.damsclient.assign_data_product(input_resource_id=data_process_id, data_product_id=data_product2_id) self.addCleanup(self.damsclient.unassign_data_product, data_process_id, data_product2_id) #Traverse through the relationships to get the links between objects res = self.dpmsclient.get_data_product_provenance_report(data_product2_id) #Make sure there are four keys self.assertEqual(len(res.keys()), 4) parent_count = 0 config_count = 0 for v in res.itervalues(): if 'parent' in v: parent_count += 1 if 'config' in v: config_count += 1 #Make sure there are three parents and four configs self.assertEqual(parent_count, 3) self.assertEqual(config_count, 4) @unittest.skip('This test is obsolete with new framework') def test_get_provenance(self): #create a deployment with metadata and an initial site and device instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "") log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id)) # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id)) self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id) # Create InstrumentAgent parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict' ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [parsed_config] ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) #------------------------------- # Create CTD Parsed data product #------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) #------------------------------- # create a data product for the site to pass the OMS check.... we need to remove this check #------------------------------- dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id) #------------------------------- # Deploy instrument device to instrument site #------------------------------- deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id) self.imsclient.deploy_instrument_device(instDevice_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) self.omsclient.activate_deployment(deployment_id) inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False) log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) ) #------------------------------- # Create the agent instance #------------------------------- port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) log.debug("TestDataProductProvenance: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) log.debug("TestDataProductProvenance: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' ) outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' ) outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id) log.debug("TestDataProductProvenance: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id) log.debug("TestDataProductProvenance: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' ) log.debug("TestDataProductProvenance: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) log.debug("TestDataProductProvenance: create output data product L2 Density") # ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, # name='L2_Density', # description='transform output pressure', # temporal_domain = tdom, # spatial_domain = sdom) # # ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, # outgoing_stream_l2_density_id, # parameter_dictionary) contactInfo = ContactInformation() contactInfo.individual_names_given = "Bill" contactInfo.individual_name_family = "Smith" contactInfo.street_address = "111 First St" contactInfo.city = "San Diego" contactInfo.email = "*****@*****.**" contactInfo.phones = ["858-555-6666"] contactInfo.country = "USA" contactInfo.postal_code = "92123" ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', contacts = [contactInfo], iso_topic_category = "my_iso_topic_category_here", quality_control_level = "1", temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L0 all data_process start") try: input_data_products = [ctd_parsed_data_product] output_data_products = [ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id, ctd_l0_temperature_output_dp_id] ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L0_all_dprocdef_id, in_data_product_ids = input_data_products, out_data_product_ids = output_data_products ) #activate only this data process just for coverage self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) log.debug("TestDataProductProvenance: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L1_conductivity_dprocdef_id, in_data_product_ids = [ctd_l0_conductivity_output_dp_id], out_data_product_ids = [ctd_l1_conductivity_output_dp_id]) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L1_pressure_dprocdef_id, in_data_product_ids = [ctd_l0_pressure_output_dp_id], out_data_product_ids = [ctd_l1_pressure_output_dp_id]) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L1_temperature_dprocdef_id, in_data_product_ids = [ctd_l0_temperature_output_dp_id], out_data_product_ids = [ctd_l1_temperature_output_dp_id]) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L2_salinity_dprocdef_id, in_data_product_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], out_data_product_ids = [ctd_l2_salinity_output_dp_id]) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_Density data_process start") try: in_dp_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id] out_dp_ids = [ctd_l2_density_output_dp_id] l2_density_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L2_density_dprocdef_id, in_data_product_ids = in_dp_ids, out_data_product_ids = out_dp_ids) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. # self._ia_client = ResourceAgentClient('iaclient', name=ResourceAgentClient._get_agent_process_id(instDevice_id, process=FakeProcess()) # print 'activate_instrument: got ia client %s', self._ia_client # log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id) self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id) self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id) self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the provenance info for the ctd density data product #------------------------------- provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id) log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict)) #validate that products are represented self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)]) density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id]) #------------------------------- # Retrieve the extended resource for this data product #------------------------------- extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id) self.assertEqual(1, len(extended_product.data_processes) ) self.assertEqual(3, len(extended_product.process_input_data_products) ) # log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list)) # log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes)) # log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products)) # log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value)) #------------------------------- # Retrieve the extended resource for this data process #------------------------------- extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) # log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def)) # log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes)) # log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products)) self.assertEqual(1, len(extended_process_def.data_processes) ) self.assertEqual(3, len(extended_process_def.output_stream_definitions) ) self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process #------------------------------- # Request the xml report #------------------------------- results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id) print results #------------------------------- # Cleanup #------------------------------- self.dpmsclient.delete_data_product(ctd_parsed_data_product) self.dpmsclient.delete_data_product(log_data_product_id) self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) print 'started services' # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain = tdom, spatial_domain = sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent[0].name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension(platform_device_id) self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(StatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": {"favorite_color": "attr desc goes here"} })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) print "Saved config:" print snap_obj.content #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config(self): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC pconfig_builder = PlatformAgentConfigurationBuilder(clients) iconfig_builder = InstrumentAgentConfigurationBuilder(clients) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_id = self.RR2.create(any_old(RT.Org)) inst_startup_config = {'startup': 'config'} required_config_keys = [ 'org_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'alarm_defs', 'children'] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertEqual({'resource_id': device_id}, config['agent']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('stream_config', config) for key in ['alarm_defs', 'children']: self.assertEqual({}, config[key]) def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config']) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('stream_config', config) if None is inst_device_id: for key in ['alarm_defs', 'children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['alarm_defs', 'startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('stream_config', config) for key in ['alarm_defs', 'startup_config']: self.assertEqual({}, config[key]) self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config}) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug("Testing that preparing a launcher without agent instance raises an error") self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False) log.debug("Making the structure for a platform agent, which will be the child") platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure() platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj) child_config = pconfig_builder.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug("Making the structure for a platform agent, which will be the parent") platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure() platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id) child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure() instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id) log.debug("Testing instrument config") iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj) instrument_config = iconfig_builder.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) full_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id) #self.fail(parent_config) #plauncher.prepare(will_launch=False) def sample_nested_platform_agent_instance_config(self): """ for informational purposes """ ret = {'org_name': 'Org_1', 'alarm_defs': {}, 'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)}, 'stream_config': {'parameter_dictionary': 'lots of stuff'}, 'agent': {'resource_id': '33e54106c4444444862da082098bc123'}, 'startup_config': {}, 'device_type': 'PlatformDevice', 'children': {'76a39596eeff4fd5b409c4cb93f0e581': {'org_name': 'Org_1', 'alarm_defs': {}, 'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)}, 'stream_config': {'parameter_dictionary': 'lots of stuff'}, 'agent': {'resource_id': '76a39596eeff4fd5b409c4cb93f0e581'}, 'startup_config': {}, 'device_type': 'PlatformDevice', 'children': {}}}} return ret